10-24-2010 08:40 PM
Hi Guys,
I am having some problem with one of the SAP services. I am unable to bring the service online via VCS. When I tried bring it up through SAP command line, it works. But VCS doesn't detect it online. It still showed offline, thus the whole cluster is in partial status. Below is the error when I tried to online with VCS:
V-16-2-13001
(CSPM4PIS) Resource(DVEBMGS02_SAPNW04_1): Output of the completed operation (clean)
Undefined subroutine &Sys::UnlinkFilesAsUser called at /opt/VRTSagents/ha/bin/SAPNW04/SAPNW04.pm line 1361.
Had[6940]: [ID 702911 daemon.notice] VCS ERROR V-16-1-20073 (CSPM4PIS) SAPNW04:DVEBMGS02_SAPNW04_1:clean:<SAPNW04::Kill> Failed to give SIGINT to sapstart. Proceeding with SIGKILL
Any idea what's goin on?
rgds,
Ronny
10-24-2010 09:09 PM
Can you paste the configuration of SAP resource from main.cf file ?
Above symptoms clearly show that monitor script of VCS is failing to detect SAP services online, need to look in detail through monitor script to know what it wants & which conditions are not getting met..
Gaurav
10-24-2010 09:14 PM
As below. Just to inform you, we have 3 SAP instance and vcs is able to detect 2 of them, except the last.
### BEGIN LOG - DATE: 101025, TIME: 121206 ###
cat main.cf
include "types.cf"
include "Db2udbTypes.cf"
include "SAPNW04Types.cf"
cluster pis_clus01 (
UserNames = { admin = gNJkLHnHOnNMiLOk, gcis = iJHgGDjHJdHFfDKhJRiH,
sapclus = cOPoONoLPlPInXPjPMoL,
sysadm = DjkCkiIpjDkiIojOjeIrjJ,
privasia = hopOllOlkSpiKipJktOl }
ClusterAddress = "172.21.36.86"
Administrators = { admin, gcis, privasia }
Operators = { sapclus, sysadm, privasia }
)
system CSPM4PIS (
)
system TSHM4PIS (
)
group ClusterService (
SystemList = { CSPM4PIS = 0, TSHM4PIS = 1 }
AutoStartList = { CSPM4PIS, TSHM4PIS }
Administrators = { privasia }
Operators = { sapclus, sysadm, privasia }
OnlineRetryLimit = 3
OnlineRetryInterval = 120
)
IPMultiNICB webip (
BaseResName = MultiNICB_pis
Address = "172.21.36.86"
NetMask = "255.255.0.0"
)
Proxy csgnic (
TargetResName = MultiNICB_pis
)
VRTSWebApp VCSweb (
Critical = 0
AppName = cmc
InstallDir = "/opt/VRTSweb/VERITAS"
TimeForOnline = 5
RestartLimit = 3
)
VCSweb requires webip
webip requires csgnic
// resource dependency tree
//
// group ClusterService
// {
// VRTSWebApp VCSweb
// {
// IPMultiNICB webip
// {
// Proxy csgnic
// }
// }
// }
group pisIPMP (
SystemList = { CSPM4PIS = 0, TSHM4PIS = 1 }
Parallel = 1
AutoStartList = { CSPM4PIS, TSHM4PIS }
Administrators = { privasia }
Operators = { sapclus, sysadm, privasia }
)
IPMultiNICB IPMultiNICB_pis (
BaseResName = MultiNICB_pis
Address @CSPM4PIS = "172.21.33.43"
Address @TSHM4PIS = "172.21.35.24"
NetMask = "255.255.0.0"
)
MultiNICB MultiNICB_pis (
MpathdCommand = "/usr/lib/inet/in.mpathd -a"
Device = { e1000g0 = 0, nxge0 = 1 }
NetworkHosts = { "172.21.52.1", "172.21.52.2", "172.21.52.11",
"172.21.52.12",
"172.21.150.22",
"172.21.150.23" }
LinkTestRatio = 3
IgnoreLinkStatus = 0
NetworkTimeout = 300
NoBroadcast = 1
DefaultRouter = "172.21.150.22"
Failback = 1
)
IPMultiNICB_pis requires MultiNICB_pis
// resource dependency tree
//
// group pisIPMP
// {
// IPMultiNICB IPMultiNICB_pis
// {
// MultiNICB MultiNICB_pis
// }
// }
group sap_db2 (
SystemList = { CSPM4PIS = 0, TSHM4PIS = 1 }
AutoStartList = { CSPM4PIS, TSHM4PIS }
Administrators = { privasia }
Operators = { sapclus, sysadm, privasia }
OnlineRetryLimit = 1
)
Db2udb xip_Db2udb (
DB2InstOwner = db2xip
DB2InstHome = "/db2/db2xip"
DatabaseName = XIP
)
DiskGroup DiskGroup_db2 (
DiskGroup = piprddb2dg
)
DiskGroup DiskGroup_sap (
DiskGroup = piprdsapdg
)
IPMultiNICB IPMultiNICB_db2 (
BaseResName = MultiNICB_pis
Address = "172.21.36.88"
NetMask = "255.255.0.0"
)
Mount Mount_IBM (
MountPoint = "/opt/IBM"
BlockDevice = "/dev/vx/dsk/piprddb2dg/IBMvol"
FSType = vxfs
FsckOpt = "-y"
)
Mount Mount_XIP (
MountPoint = "/usr/sap/XIP"
BlockDevice = "/dev/vx/dsk/piprdsapdg/xipvol"
FSType = vxfs
FsckOpt = "-y"
)
Mount Mount_db2dump (
MountPoint = "/db2/XIP/db2dump"
BlockDevice = "/dev/vx/dsk/piprddb2dg/db2dumpvol"
FSType = vxfs
FsckOpt = "-y"
)
Mount Mount_db2xip (
MountPoint = "/db2/db2xip"
BlockDevice = "/dev/vx/dsk/piprddb2dg/db2xipvol"
FSType = vxfs
FsckOpt = "-y"
)
Mount Mount_log_archive (
MountPoint = "/db2/XIP/log_archive"
BlockDevice = "/dev/vx/dsk/piprddb2dg/logarchvol"
FSType = vxfs
FsckOpt = "-y"
)
Mount Mount_log_dir (
MountPoint = "/db2/XIP/log_dir"
BlockDevice = "/dev/vx/dsk/piprddb2dg/logdirvol"
FSType = vxfs
FsckOpt = "-y"
)
Mount Mount_pxip (
MountPoint = "/db2/XIP/"
BlockDevice = "/dev/vx/dsk/piprddb2dg/pxipvol"
FSType = vxfs
FsckOpt = "-y"
)
Mount Mount_sapdata1 (
MountPoint = "/db2/XIP/sapdata1"
BlockDevice = "/dev/vx/dsk/piprddb2dg/sapdatavol1"
FSType = vxfs
FsckOpt = "-y"
)
Mount Mount_sapdata2 (
MountPoint = "/db2/XIP/sapdata2"
BlockDevice = "/dev/vx/dsk/piprddb2dg/sapdatavol2"
FSType = vxfs
FsckOpt = "-y"
)
Mount Mount_sapdata3 (
MountPoint = "/db2/XIP/sapdata3"
BlockDevice = "/dev/vx/dsk/piprddb2dg/sapdatavol3"
FSType = vxfs
FsckOpt = "-y"
)
Mount Mount_sapdata4 (
MountPoint = "/db2/XIP/sapdata4"
BlockDevice = "/dev/vx/dsk/piprddb2dg/sapdatavol4"
FSType = vxfs
FsckOpt = "-y"
)
Mount Mount_sapdata5 (
MountPoint = "/db2/XIP/sapdata5"
BlockDevice = "/dev/vx/dsk/piprddb2dg/sapdatavol5"
FSType = vxfs
FsckOpt = "-y"
)
Mount Mount_sapdata6 (
MountPoint = "/db2/XIP/sapdata6"
BlockDevice = "/dev/vx/dsk/piprddb2dg/sapdatavol6"
FSType = vxfs
FsckOpt = "-y"
)
Mount Mount_sapmnt (
MountPoint = "/sapmnt/XIP"
BlockDevice = "/dev/vx/dsk/piprdsapdg/sapmntvol"
FSType = vxfs
FsckOpt = "-y"
)
Mount Mount_saptemp1 (
MountPoint = "/db2/XIP/saptemp1"
BlockDevice = "/dev/vx/dsk/piprddb2dg/saptemp1vol"
FSType = vxfs
FsckOpt = "-y"
)
Mount Mount_trans (
MountPoint = "/usr/sap/trans"
BlockDevice = "/dev/vx/dsk/piprdsapdg/transvol"
FSType = vxfs
FsckOpt = "-y"
)
Proxy Proxy-db2 (
TargetResName = MultiNICB_pis
)
SAPNW04 ASCS_SAPNW04_1 (
EnvFile = "/sapmnt/XIP/VCS_sap/sapxipVCS.env"
InstName = ASCS01
InstType = ENQUEUE
ProcMon = "en ms"
SAPAdmin = xipadm
SAPMonHome = "/usr/sap/XIP/SYS/exe/run"
SAPSID = XIP
StartProfile = "/usr/sap/XIP/SYS/profile/START_ASCS01_PISWESTP"
)
SAPNW04 DVEBMGS02_SAPNW04_1 (
EnvFile = "/sapmnt/XIP/VCS_sap/sapxipVCS.env"
InstName = DVEBMGS02
ProcMon = "dw ig se co"
SAPAdmin = xipadm
SAPMonHome = "/usr/sap/XIP/SYS/exe/run"
SAPSID = XIP
StartProfile = "/usr/sap/XIP/SYS/profile/START_DVEBMGS02_PISWESTP"
)
SAPNW04 SCS_SAPNW04_1 (
EnvFile = "/sapmnt/XIP/VCS_sap/sapxipVCS.env"
InstName = SCS00
InstType = ENQUEUE
ProcMon = "ms en"
SAPAdmin = xipadm
SAPMonHome = "/usr/sap/XIP/SYS/exe/run"
SAPSID = XIP
StartProfile = "/usr/sap/XIP/SYS/profile/START_SCS00_PISWESTP"
)
Volume Volume_IBMvol (
Volume = IBMvol
DiskGroup = piprddb2dg
)
Volume Volume_db2dumpvol (
Volume = logarchvol
DiskGroup = piprddb2dg
)
Volume Volume_logarchvol (
Volume = logarchvol
DiskGroup = piprddb2dg
)
Volume Volume_logdirvol (
Volume = logarchvol
DiskGroup = piprddb2dg
)
Volume Volume_pxipvol (
Volume = pxipvol
DiskGroup = piprddb2dg
)
Volume Volume_sapdatavol1 (
Volume = logarchvol
DiskGroup = piprddb2dg
)
Volume Volume_sapdatavol2 (
Volume = logarchvol
DiskGroup = piprddb2dg
)
Volume Volume_sapdatavol3 (
Volume = logarchvol
DiskGroup = piprddb2dg
)
Volume Volume_sapdatavol4 (
Volume = logarchvol
DiskGroup = piprddb2dg
)
Volume Volume_sapdatavol5 (
Volume = logarchvol
DiskGroup = piprddb2dg
)
Volume Volume_sapdatavol6 (
Volume = logarchvol
DiskGroup = piprddb2dg
)
Volume Volume_sapmntvol (
Volume = transvol
DiskGroup = piprdsapdg
)
Volume Volume_saptemp1vol (
Volume = saptemp1vol
DiskGroup = piprddb2dg
)
Volume Volume_transvol (
Volume = transvol
DiskGroup = piprdsapdg
)
Volume Volume_xipvol (
Volume = xipvol
DiskGroup = piprdsapdg
)
Volume db2xipvol (
Volume = logarchvol
DiskGroup = piprddb2dg
)
ASCS_SAPNW04_1 requires SCS_SAPNW04_1
DVEBMGS02_SAPNW04_1 requires ASCS_SAPNW04_1
IPMultiNICB_db2 requires Proxy-db2
Mount_IBM requires Mount_pxip
Mount_IBM requires Volume_IBMvol
Mount_XIP requires Volume_xipvol
Mount_db2dump requires Mount_pxip
Mount_db2dump requires Volume_db2dumpvol
Mount_db2xip requires Mount_pxip
Mount_db2xip requires db2xipvol
Mount_log_archive requires Mount_pxip
Mount_log_archive requires Volume_logarchvol
Mount_log_dir requires Mount_pxip
Mount_log_dir requires Volume_logdirvol
Mount_pxip requires Volume_pxipvol
Mount_sapdata1 requires Mount_pxip
Mount_sapdata1 requires Volume_sapdatavol1
Mount_sapdata2 requires Mount_pxip
Mount_sapdata2 requires Volume_sapdatavol2
Mount_sapdata3 requires Mount_pxip
Mount_sapdata3 requires Volume_sapdatavol3
Mount_sapdata4 requires Mount_pxip
Mount_sapdata4 requires Volume_sapdatavol4
Mount_sapdata5 requires Mount_pxip
Mount_sapdata5 requires Volume_sapdatavol5
Mount_sapdata6 requires Mount_pxip
Mount_sapdata6 requires Volume_sapdatavol6
Mount_sapmnt requires Volume_sapmntvol
Mount_saptemp1 requires Mount_pxip
Mount_saptemp1 requires Volume_saptemp1vol
Mount_trans requires Volume_transvol
SCS_SAPNW04_1 requires xip_Db2udb
Volume_IBMvol requires DiskGroup_db2
Volume_db2dumpvol requires DiskGroup_db2
Volume_logarchvol requires DiskGroup_db2
Volume_logdirvol requires DiskGroup_db2
Volume_pxipvol requires DiskGroup_db2
Volume_sapdatavol1 requires DiskGroup_db2
Volume_sapdatavol2 requires DiskGroup_db2
Volume_sapdatavol3 requires DiskGroup_db2
Volume_sapdatavol4 requires DiskGroup_db2
Volume_sapdatavol5 requires DiskGroup_db2
Volume_sapdatavol6 requires DiskGroup_db2
Volume_sapmntvol requires DiskGroup_sap
Volume_saptemp1vol requires DiskGroup_db2
Volume_transvol requires DiskGroup_sap
Volume_xipvol requires DiskGroup_sap
db2xipvol requires DiskGroup_db2
xip_Db2udb requires IPMultiNICB_db2
xip_Db2udb requires Mount_IBM
xip_Db2udb requires Mount_XIP
xip_Db2udb requires Mount_db2dump
xip_Db2udb requires Mount_db2xip
xip_Db2udb requires Mount_log_archive
xip_Db2udb requires Mount_log_dir
xip_Db2udb requires Mount_pxip
xip_Db2udb requires Mount_sapdata1
xip_Db2udb requires Mount_sapdata2
xip_Db2udb requires Mount_sapdata3
xip_Db2udb requires Mount_sapdata4
xip_Db2udb requires Mount_sapdata5
xip_Db2udb requires Mount_sapdata6
xip_Db2udb requires Mount_sapmnt
xip_Db2udb requires Mount_saptemp1
xip_Db2udb requires Mount_trans
// resource dependency tree
//
// group sap_db2
// {
// SAPNW04 DVEBMGS02_SAPNW04_1
// {
// SAPNW04 ASCS_SAPNW04_1
// {
// SAPNW04 SCS_SAPNW04_1
// {
// Db2udb xip_Db2udb
// {
// IPMultiNICB IPMultiNICB_db2
// {
// Proxy Proxy-db2
// }
// Mount Mount_IBM
// {
// Mount Mount_pxip
// {
// Volume Volume_pxipvol
// {
// DiskGroup DiskGroup_db2
// }
// }
// Volume Volume_IBMvol
// {
// DiskGroup DiskGroup_db2
// }
// }
// Mount Mount_XIP
// {
// Volume Volume_xipvol
// {
// DiskGroup DiskGroup_sap
// }
// }
// Mount Mount_db2dump
// {
// Mount Mount_pxip
// {
// Volume Volume_pxipvol
// {
// DiskGroup DiskGroup_db2
// }
// }
// Volume Volume_db2dumpvol
// {
// DiskGroup DiskGroup_db2
// }
// }
// Mount Mount_db2xip
// {
// Mount Mount_pxip
// {
// Volume Volume_pxipvol
// {
// DiskGroup DiskGroup_db2
// }
// }
// Volume db2xipvol
// {
// DiskGroup DiskGroup_db2
// }
// }
// Mount Mount_log_archive
// {
// Mount Mount_pxip
// {
// Volume Volume_pxipvol
// {
// DiskGroup DiskGroup_db2
// }
// }
// Volume Volume_logarchvol
// {
// DiskGroup DiskGroup_db2
// }
// }
// Mount Mount_log_dir
// {
// Mount Mount_pxip
// {
// Volume Volume_pxipvol
// {
// DiskGroup DiskGroup_db2
// }
// }
// Volume Volume_logdirvol
// {
// DiskGroup DiskGroup_db2
// }
// }
// Mount Mount_pxip
// {
// Volume Volume_pxipvol
// {
// DiskGroup DiskGroup_db2
// }
// }
// Mount Mount_sapdata1
// {
// Mount Mount_pxip
// {
// Volume Volume_pxipvol
// {
// DiskGroup DiskGroup_db2
// }
// }
// Volume Volume_sapdatavol1
// {
// DiskGroup DiskGroup_db2
// }
// }
// Mount Mount_sapdata2
// {
// Mount Mount_pxip
// {
// Volume Volume_pxipvol
// {
// DiskGroup DiskGroup_db2
// }
// }
// Volume Volume_sapdatavol2
// {
// DiskGroup DiskGroup_db2
// }
// }
// Mount Mount_sapdata3
// {
// Mount Mount_pxip
// {
// Volume Volume_pxipvol
// {
// DiskGroup DiskGroup_db2
// }
// }
// Volume Volume_sapdatavol3
// {
// DiskGroup DiskGroup_db2
// }
// }
// Mount Mount_sapdata4
// {
// Mount Mount_pxip
// {
// Volume Volume_pxipvol
// {
// DiskGroup DiskGroup_db2
// }
// }
// Volume Volume_sapdatavol4
// {
// DiskGroup DiskGroup_db2
// }
// }
// Mount Mount_sapdata5
// {
// Mount Mount_pxip
// {
// Volume Volume_pxipvol
// {
// DiskGroup DiskGroup_db2
// }
// }
// Volume Volume_sapdatavol5
// {
// DiskGroup DiskGroup_db2
// }
// }
// Mount Mount_sapdata6
// {
// Mount Mount_pxip
// {
// Volume Volume_pxipvol
// {
// DiskGroup DiskGroup_db2
// }
// }
// Volume Volume_sapdatavol6
// {
// DiskGroup DiskGroup_db2
// }
// }
// Mount Mount_sapmnt
// {
// Volume Volume_sapmntvol
// {
// DiskGroup DiskGroup_sap
// }
// }
// Mount Mount_saptemp1
// {
// Mount Mount_pxip
// {
// Volume Volume_pxipvol
// {
// DiskGroup DiskGroup_db2
// }
// }
// Volume Volume_saptemp1vol
// {
// DiskGroup DiskGroup_db2
// }
// }
// Mount Mount_trans
// {
// Volume Volume_transvol
// {
// DiskGroup DiskGroup_sap
// }
// }
// }
// }
// }
// }
// }
bash-3.00#
### END LOG - DATE: 101025, TIME: 121228 ###