Forum Discussion
fxcva
15 years agoLevel 2
Hello again:
first thanks for your comments and help
and yes VCSweb is online on both nodes, Just tried and it worked on both nodes.
Could that be the origin of the problem?
Her are the outputs from the command or files
hares -display VCSWeb
(root_mxcpm9kh1101dc4)@/ # hares -display VCSWeb
#Resource Attribute System Value
VCSWeb Group global ClusterService
VCSWeb Type global VRTSWebApp
VCSWeb AutoStart global 1
VCSWeb Critical global 0
VCSWeb Enabled global 1
VCSWeb LastOnline global mxcpm9kh1401dc4
VCSWeb MonitorOnly global 0
VCSWeb ResourceOwner global unknown
VCSWeb TriggerEvent global 0
VCSWeb ArgListValues mxcpm9kh1101dc4 AppName 1 cmc InstallDir 1 /opt/VRTSweb/VERITAS TimeForOnline 1 0
VCSWeb ArgListValues mxcpm9kh1401dc4 AppName 1 cmc InstallDir 1 /opt/VRTSweb/VERITAS TimeForOnline 1 0
VCSWeb ConfidenceLevel mxcpm9kh1101dc4 100
VCSWeb ConfidenceLevel mxcpm9kh1401dc4 100
VCSWeb Flags mxcpm9kh1101dc4
VCSWeb Flags mxcpm9kh1401dc4
VCSWeb IState mxcpm9kh1101dc4 not waiting
VCSWeb IState mxcpm9kh1401dc4 not waiting
VCSWeb Probed mxcpm9kh1101dc4 1
VCSWeb Probed mxcpm9kh1401dc4 1
VCSWeb Start mxcpm9kh1101dc4 1
VCSWeb Start mxcpm9kh1401dc4 1
VCSWeb State mxcpm9kh1101dc4 ONLINE
VCSWeb State mxcpm9kh1401dc4 ONLINE
VCSWeb AppName global cmc
VCSWeb ComputeStats global 0
VCSWeb InstallDir global /opt/VRTSweb/VERITAS
VCSWeb TimeForOnline global 0
VCSWeb MonitorTimeStats mxcpm9kh1101dc4 Avg 0 TS
VCSWeb MonitorTimeStats mxcpm9kh1401dc4 Avg 0 TS
VCSWeb ResourceInfo mxcpm9kh1101dc4 State Valid Msg TS
VCSWeb ResourceInfo mxcpm9kh1401dc4 State Valid Msg TS
here is main.cf
(root_mxcpm9kh1101dc4)@/etc/VRTSvcs/conf/config # more main.cf
include "OracleASMTypes.cf"
include "types.cf"
include "CFSTypes.cf"
include "CVMTypes.cf"
include "MultiPrivNIC.cf"
include "OracleTypes.cf"
include "PrivNIC.cf"
cluster VCS_CH_DC04 (
UserNames = { admin = dlmElgLimHmmKumGlj, stuvcs = ePPlPRoSQjQOoSQmPK }
Administrators = { admin, stuvcs }
UseFence = SCSI3
HacliUserLevel = COMMANDROOT
)
system mxcpm9kh1101dc4 (
)
system mxcpm9kh1401dc4 (
)
group ClusterService (
SystemList = { mxcpm9kh1101dc4 = 0, mxcpm9kh1401dc4 = 1 }
Parallel = 1
AutoStartList = { mxcpm9kh1101dc4, mxcpm9kh1401dc4 }
)
VRTSWebApp VCSWeb (
Critical = 0
AppName = cmc
InstallDir = "/opt/VRTSweb/VERITAS"
)
group cvm (
SystemList = { mxcpm9kh1101dc4 = 0, mxcpm9kh1401dc4 = 1 }
AutoFailOver = 0
Parallel = 1
AutoStartList = { mxcpm9kh1101dc4, mxcpm9kh1401dc4 }
)
Application cssd (
Critical = 0
StartProgram = "/opt/VRTSvcs/rac/bin/cssd-online"
StopProgram = "/opt/VRTSvcs/rac/bin/cssd-offline"
CleanProgram = "/opt/VRTSvcs/rac/bin/cssd-clean"
MonitorProgram = "/opt/VRTSvcs/rac/bin/cssd-monitor"
)
CFSMount ORCRS_MNT_CRS (
Critical = 0
MountPoint = "/crs_vol"
BlockDevice = "/dev/vx/dsk/crs_dg/crs_vol"
MountOpt = "mincache=direct"
)
CFSfsckd vxfsckd (
Critical = 0
)
CVMCluster cvm_clus (
Critical = 0
CVMClustName = VCS_CH_DC04
CVMNodeId = { mxcpm9kh1101dc4 = 0, mxcpm9kh1401dc4 = 1 }
CVMTransport = gab
CVMTimeout = 200
)
CVMVolDg ORCRS_DG (
Critical = 0
CVMDiskGroup = crs_dg
CVMVolume = { crs_vol }
CVMActivation = sw
)
CVMVxconfigd cvm_vxconfigd (
Critical = 0
CVMVxconfigdArgs = { syslog }
)
PrivNIC ora_priv (
Critical = 0
Device = { nxge1 = 0, nxge5 = 1 }
Address @mxcpm9kh1101dc4 = "10.10.235.1"
Address @mxcpm9kh1401dc4 = "10.10.235.2"
NetMask = "255.255.255.240"
)
ORCRS_DG requires cvm_clus
ORCRS_MNT_CRS requires ORCRS_DG
ORCRS_MNT_CRS requires vxfsckd
cssd requires ORCRS_MNT_CRS
cssd requires ora_priv
cvm_clus requires cvm_vxconfigd
vxfsckd requires cvm_clus
group ora_arch (
SystemList = { mxcpm9kh1101dc4 = 0, mxcpm9kh1401dc4 = 1 }
Parallel = 1
AutoStartList = { mxcpm9kh1101dc4, mxcpm9kh1401dc4 }
)
CFSMount ORARC_MNT_U05 (
Critical = 0
MountPoint = "/u05"
BlockDevice = "/dev/vx/dsk/archives_dg/u05"
MountOpt = largefiles
)
CFSMount ORREDO_MNT_U02 (
Critical = 0
MountPoint = "/u02"
BlockDevice = "/dev/vx/dsk/redolog_dg/u02"
MountOpt = largefiles
)
CFSMount ORREDO_MNT_U03 (
Critical = 0
MountPoint = "/u03"
BlockDevice = "/dev/vx/dsk/redolog_dg/u03"
MountOpt = largefiles
)
CFSMount ORREDO_MNT_U04 (
Critical = 0
MountPoint = "/u04"
BlockDevice = "/dev/vx/dsk/redolog_dg/u04"
MountOpt = largefiles
)
CVMVolDg ORARC_DG (
Critical = 0
CVMDiskGroup = archives_dg
CVMVolume = { u05 }
CVMActivation = sw
)
CVMVolDg ORREDO_DG (
Critical = 0
CVMDiskGroup = redolog_dg
CVMVolume = { u02, u03, u04 }
CVMActivation = sw
)
requires group cvm online local firm
ORARC_MNT_U05 requires ORARC_DG
ORREDO_MNT_U02 requires ORREDO_DG
ORREDO_MNT_U03 requires ORREDO_DG
ORREDO_MNT_U04 requires ORREDO_DG
group ora_crystwb (
SystemList = { mxcpm9kh1101dc4 = 0, mxcpm9kh1401dc4 = 1 }
Parallel = 1
AutoStartList = { mxcpm9kh1101dc4, mxcpm9kh1401dc4 }
)
CFSMount CRYSTWB_MNT_u07 (
Critical = 0
MountPoint = "/u07"
BlockDevice = "/dev/vx/dsk/crystwb_dg/u07"
)
CVMVolDg CRYSTWB_DG (
Critical = 0
CVMDiskGroup = crystwb_dg
CVMVolume = { u07 }
CVMActivation = sw
)
Oracle CRYSTWB_ORA (
Critical = 0
Sid @mxcpm9kh1101dc4 = crystwb1
Sid @mxcpm9kh1401dc4 = crystwb2
Owner = oracle
Home = "/u01/sq/ora_1/oracle/db/10.2.0"
Pfile @mxcpm9kh1101dc4 = "/u01/sq/ora_1/oracle/db/10.2.0/dbs/initcrystwb1.ora"
Pfile @mxcpm9kh1401dc4 = "/u01/sq/ora_1/oracle/db/10.2.0/dbs/initcrystwb2.ora"
)
requires group ora_arch online local firm
CRYSTWB_MNT_u07 requires CRYSTWB_DG
CRYSTWB_ORA requires CRYSTWB_MNT_u07
group ora_danpro (
SystemList = { mxcpm9kh1101dc4 = 0, mxcpm9kh1401dc4 = 1 }
Parallel = 1
AutoStartList = { mxcpm9kh1101dc4, mxcpm9kh1401dc4 }
)
CFSMount DANPRO_MNT_danpro2 (
Critical = 0
MountPoint = "/danpro2"
BlockDevice = "/dev/vx/dsk/danpro2_dg/danpro2"
)
CFSMount DANPRO_MNT_u100 (
Critical = 0
MountPoint = "/u100"
BlockDevice = "/dev/vx/dsk/oraseg1_dg/u100"
MountOpt = largefiles
)
CFSMount DANPRO_MNT_u101 (
Critical = 0
MountPoint = "/u101"
BlockDevice = "/dev/vx/dsk/oraseg1_dg/u101"
MountOpt = largefiles
)
CFSMount DANPRO_MNT_u102 (
Critical = 0
MountPoint = "/u102"
BlockDevice = "/dev/vx/dsk/oraseg1_dg/u102"
MountOpt = largefiles
)
CFSMount DANPRO_MNT_u103 (
Critical = 0
MountPoint = "/u103"
BlockDevice = "/dev/vx/dsk/oraseg1_dg/u103"
MountOpt = largefiles
)
CFSMount DANPRO_MNT_u104 (
Critical = 0
MountPoint = "/u104"
BlockDevice = "/dev/vx/dsk/oraseg1_dg/u104"
MountOpt = largefiles
)
CFSMount DANPRO_MNT_u105 (
Critical = 0
MountPoint = "/u105"
BlockDevice = "/dev/vx/dsk/oraseg1_dg/u105"
MountOpt = largefiles
)
CFSMount DANPRO_MNT_u106 (
Critical = 0
MountPoint = "/u106"
BlockDevice = "/dev/vx/dsk/oraseg1_dg/u106"
MountOpt = largefiles
)
CFSMount DANPRO_MNT_u107 (
Critical = 0
MountPoint = "/u107"
BlockDevice = "/dev/vx/dsk/oraseg1_dg/u107"
MountOpt = largefiles
)
CFSMount DANPRO_MNT_u108 (
Critical = 0
MountPoint = "/u108"
BlockDevice = "/dev/vx/dsk/oraseg1_dg/u108"
MountOpt = largefiles
)
CFSMount DANPRO_MNT_u109 (
Critical = 0
MountPoint = "/u109"
BlockDevice = "/dev/vx/dsk/oraseg1_dg/u109"
MountOpt = largefiles
)
CFSMount DANPRO_MNT_u110 (
Critical = 0
MountPoint = "/u110"
BlockDevice = "/dev/vx/dsk/oraseg1_dg/u110"
MountOpt = largefiles
)
CFSMount DANPRO_MNT_u111 (
Critical = 0
MountPoint = "/u111"
BlockDevice = "/dev/vx/dsk/oraseg1_dg/u111"
MountOpt = largefiles
)
CFSMount DANPRO_MNT_u112 (
Critical = 0
MountPoint = "/u112"
BlockDevice = "/dev/vx/dsk/oraseg1_dg/u112"
MountOpt = largefiles
)
CFSMount DANPRO_MNT_u113 (
Critical = 0
MountPoint = "/u113"
BlockDevice = "/dev/vx/dsk/oraseg1_dg/u113"
MountOpt = largefiles
)
CFSMount DANPRO_MNT_u114 (
Critical = 0
MountPoint = "/u114"
BlockDevice = "/dev/vx/dsk/oraseg1_dg/u114"
MountOpt = largefiles
)
CFSMount DANPRO_MNT_u115 (
Critical = 0
MountPoint = "/u115"
BlockDevice = "/dev/vx/dsk/oraseg1_dg/u115"
MountOpt = largefiles
)
CFSMount DANPRO_MNT_u116 (
Critical = 0
MountPoint = "/u116"
BlockDevice = "/dev/vx/dsk/oraseg1_dg/u116"
MountOpt = largefiles
)
CFSMount DANPRO_MNT_u117 (
Critical = 0
MountPoint = "/u117"
BlockDevice = "/dev/vx/dsk/oraseg1_dg/u117"
MountOpt = largefiles
)
CFSMount DANPRO_MNT_u118 (
Critical = 0
MountPoint = "/u118"
BlockDevice = "/dev/vx/dsk/oraseg1_dg/u118"
MountOpt = largefiles
)
CFSMount DANPRO_MNT_u119 (
Critical = 0
MountPoint = "/u119"
BlockDevice = "/dev/vx/dsk/oraseg1_dg/u119"
MountOpt = largefiles
)
CVMVolDg DANPRO2_DG (
Critical = 0
CVMDiskGroup = danpro2_dg
CVMVolume = { danpro2 }
CVMActivation = sw
)
CVMVolDg DANPRO_DG (
Critical = 0
CVMDiskGroup = oraseg1_dg
CVMVolume = {
"u100 u101 u102 u103 u104 u105 u106 u107 u108 u109 u110 u111 u112 u113 u114 u115" }
CVMActivation = sw
)
Oracle DANPRO_ORA (
Critical = 0
Sid @mxcpm9kh1101dc4 = danpro1
Sid @mxcpm9kh1401dc4 = danpro2
Owner = oracle
Home = "/u01/sq/ora_1/oracle/db/10.2.0"
Pfile @mxcpm9kh1101dc4 = "/u01/sq/ora_1/oracle/db/10.2.0/dbs/initdanpro1.ora"
Pfile @mxcpm9kh1401dc4 = "/u01/sq/ora_1/oracle/db/10.2.0/dbs/initdanpro2.ora"
)
requires group ora_arch online local firm
DANPRO_MNT_danpro2 requires DANPRO2_DG
DANPRO_MNT_u100 requires DANPRO_DG
DANPRO_MNT_u101 requires DANPRO_DG
DANPRO_MNT_u102 requires DANPRO_DG
DANPRO_MNT_u103 requires DANPRO_DG
DANPRO_MNT_u104 requires DANPRO_DG
DANPRO_MNT_u105 requires DANPRO_DG
DANPRO_MNT_u106 requires DANPRO_DG
DANPRO_MNT_u107 requires DANPRO_DG
DANPRO_MNT_u108 requires DANPRO_DG
DANPRO_MNT_u109 requires DANPRO_DG
DANPRO_MNT_u110 requires DANPRO_DG
DANPRO_MNT_u111 requires DANPRO_DG
DANPRO_MNT_u112 requires DANPRO_DG
DANPRO_MNT_u113 requires DANPRO_DG
DANPRO_MNT_u114 requires DANPRO_DG
DANPRO_MNT_u115 requires DANPRO_DG
DANPRO_MNT_u116 requires DANPRO_DG
DANPRO_MNT_u117 requires DANPRO_DG
DANPRO_MNT_u118 requires DANPRO_DG
DANPRO_MNT_u119 requires DANPRO_DG
DANPRO_ORA requires DANPRO_MNT_danpro2
DANPRO_ORA requires DANPRO_MNT_u100
DANPRO_ORA requires DANPRO_MNT_u101
DANPRO_ORA requires DANPRO_MNT_u102
DANPRO_ORA requires DANPRO_MNT_u103
DANPRO_ORA requires DANPRO_MNT_u104
DANPRO_ORA requires DANPRO_MNT_u105
DANPRO_ORA requires DANPRO_MNT_u106
DANPRO_ORA requires DANPRO_MNT_u107
DANPRO_ORA requires DANPRO_MNT_u108
DANPRO_ORA requires DANPRO_MNT_u109
DANPRO_ORA requires DANPRO_MNT_u110
DANPRO_ORA requires DANPRO_MNT_u111
DANPRO_ORA requires DANPRO_MNT_u112
DANPRO_ORA requires DANPRO_MNT_u113
DANPRO_ORA requires DANPRO_MNT_u114
DANPRO_ORA requires DANPRO_MNT_u115
DANPRO_ORA requires DANPRO_MNT_u116
DANPRO_ORA requires DANPRO_MNT_u117
DANPRO_ORA requires DANPRO_MNT_u118
DANPRO_ORA requires DANPRO_MNT_u119
group ora_faxflow (
SystemList = { mxcpm9kh1101dc4 = 0, mxcpm9kh1401dc4 = 1 }
Parallel = 1
AutoStartList = { mxcpm9kh1101dc4, mxcpm9kh1401dc4 }
)
CFSMount FAXFLOW_MNT_u09 (
Critical = 0
MountPoint = "/u09"
BlockDevice = "/dev/vx/dsk/faxflow_dg/u09"
)
CVMVolDg FAXFLOW_DG (
Critical = 0
CVMDiskGroup = faxflow_dg
CVMVolume = { u09 }
CVMActivation = sw
)
Oracle FAXFLOW_ORA (
Critical = 0
Sid @mxcpm9kh1101dc4 = faxflow11
Sid @mxcpm9kh1401dc4 = faxflow12
Owner = oracle
Home = "/u01/sq/ora_1/oracle/db/10.2.0"
Pfile @mxcpm9kh1101dc4 = "/u01/sq/ora_1/oracle/db/10.2.0/dbs/initfaxflow11.ora"
Pfile @mxcpm9kh1401dc4 = "/u01/sq/ora_1/oracle/db/10.2.0/dbs/initfaxflow12.ora"
)
requires group ora_arch online local firm
FAXFLOW_MNT_u09 requires FAXFLOW_DG
FAXFLOW_ORA requires FAXFLOW_MNT_u09
group ora_tandapro (
SystemList = { mxcpm9kh1101dc4 = 0, mxcpm9kh1401dc4 = 1 }
Parallel = 1
AutoStartList = { mxcpm9kh1101dc4, mxcpm9kh1401dc4 }
)
CFSMount TANDAPRO_MNT_u08 (
Critical = 0
MountPoint = "/u08"
BlockDevice = "/dev/vx/dsk/tandapro_dg/u08"
)
CVMVolDg TANDAPRO_DG (
Critical = 0
CVMDiskGroup = tandapro_dg
CVMVolume = { u08 }
CVMActivation = sw
)
Oracle TANDAPRO_ORA (
Critical = 0
Sid @mxcpm9kh1101dc4 = tandapro1
Sid @mxcpm9kh1401dc4 = tandapro2
Owner = oracle
Home = "/u01/sq/ora_1/oracle/db/10.2.0"
Pfile @mxcpm9kh1101dc4 = "/u01/sq/ora_1/oracle/db/10.2.0/dbs/inittandapro1.ora"
Pfile @mxcpm9kh1401dc4 = "/u01/sq/ora_1/oracle/db/10.2.0/dbs/inittandapro2.ora"
)
requires group ora_arch online local firm
TANDAPRO_MNT_u08 requires TANDAPRO_DG
TANDAPRO_ORA requires TANDAPRO_MNT_u08
_________________________________________________________________
/var/adm/messages extract from node mxcpm9kh1101dc4
Mar 8 12:55:20 mxcpm9kh1101dc4 root: [ID 702911 daemon.alert] The audit_warn mail alias is not defined
Mar 8 12:55:20 mxcpm9kh1101dc4 root: [ID 702911 daemon.alert] Soft limit exceeded in file /var/audit.
Mar 8 12:55:20 mxcpm9kh1101dc4 root: [ID 702911 daemon.alert] The audit_warn mail alias is not defined
Mar 8 12:55:20 mxcpm9kh1101dc4 root: [ID 702911 daemon.alert] Soft limit exceeded on all filesystems.
Mar 8 12:55:20 mxcpm9kh1101dc4 last message repeated 2 times
Mar 8 12:58:07 mxcpm9kh1101dc4 sshd[882]: [ID 800047 auth.notice] Failed none for root from 164.2.45.246 port 49475 ssh2
Mar 8 12:58:08 mxcpm9kh1101dc4 sshd[882]: [ID 800047 auth.info] Accepted keyboard-interactive for root from 164.2.45.246 port 49475 ssh2
Mar 8 12:59:18 mxcpm9kh1101dc4 gab: [ID 316943 kern.notice] GAB INFO V-15-1-20036 Port h gen aeab23 membership 0
Mar 8 12:59:18 mxcpm9kh1101dc4 gab: [ID 674723 kern.notice] GAB INFO V-15-1-20038 Port h gen aeab23 k_jeopardy ;1
Mar 8 12:59:18 mxcpm9kh1101dc4 gab: [ID 513393 kern.notice] GAB INFO V-15-1-20040 Port h gen aeab23 visible ;1
Mar 8 12:59:18 mxcpm9kh1101dc4 Had[26804]: [ID 702911 daemon.notice] VCS INFO V-16-1-10077 Received new cluster membership
Mar 8 12:59:18 mxcpm9kh1101dc4 Had[26804]: [ID 702911 daemon.notice] VCS ERROR V-16-1-10113 System mxcpm9kh1401dc4 (Node '1') is in DDNA Membership - Mem
bership: 0x1, Visible: 0x0
Mar 8 12:59:18 mxcpm9kh1101dc4 su: [ID 366847 auth.info] 'su oracle' succeeded for root on /dev/???
Mar 8 12:59:31 mxcpm9kh1101dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/faxflow_dg/u09 - Set Primary nodeid to 1
Mar 8 12:59:32 mxcpm9kh1101dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/tandapro_dg/u08 - Set Primary nodeid to 1
Mar 8 12:59:37 mxcpm9kh1101dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/danpro2_dg/danpro2 - Set Primary nodeid to 1
Mar 8 12:59:37 mxcpm9kh1101dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u107 - Set Primary nodeid to 1
Mar 8 12:59:37 mxcpm9kh1101dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u108 - Set Primary nodeid to 1
Mar 8 12:59:37 mxcpm9kh1101dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u106 - Set Primary nodeid to 1
Mar 8 12:59:37 mxcpm9kh1101dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u105 - Set Primary nodeid to 1
Mar 8 12:59:37 mxcpm9kh1101dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u104 - Set Primary nodeid to 1
Mar 8 12:59:37 mxcpm9kh1101dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u103 - Set Primary nodeid to 1
Mar 8 12:59:38 mxcpm9kh1101dc4 Had[26804]: [ID 702911 daemon.notice] VCS ERROR V-16-1-5503 (mxcpm9kh1101dc4) CFSMount:DANPRO_MNT_u100:offline:Umount Fail
ed : Mount Point : /u100
Mar 8 12:59:38 mxcpm9kh1101dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u102 - Set Primary nodeid to 1
Mar 8 12:59:38 mxcpm9kh1101dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u101 - Set Primary nodeid to 1
Mar 8 12:59:38 mxcpm9kh1101dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u110 - Set Primary nodeid to 1
Mar 8 12:59:38 mxcpm9kh1101dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u109 - Set Primary nodeid to 1
Mar 8 12:59:39 mxcpm9kh1101dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u111 - Set Primary nodeid to 1
Mar 8 12:59:39 mxcpm9kh1101dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u113 - Set Primary nodeid to 1
Mar 8 12:59:39 mxcpm9kh1101dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u117 - Set Primary nodeid to 1
Mar 8 12:59:39 mxcpm9kh1101dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u116 - Set Primary nodeid to 1
Mar 8 12:59:39 mxcpm9kh1101dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u114 - Set Primary nodeid to 1
Mar 8 12:59:39 mxcpm9kh1101dc4 Had[26804]: [ID 702911 daemon.notice] VCS ERROR V-16-1-5503 (mxcpm9kh1101dc4) CFSMount:DANPRO_MNT_u119:offline:Umount Fail
ed : Mount Point : /u119
Mar 8 12:59:39 mxcpm9kh1101dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u118 - Set Primary nodeid to 1
Mar 8 12:59:40 mxcpm9kh1101dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u115 - Set Primary nodeid to 1
Mar 8 12:59:40 mxcpm9kh1101dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u112 - Set Primary nodeid to 1
Mar 8 12:59:41 mxcpm9kh1101dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/crystwb_dg/u07 - Set Primary nodeid to 1
Mar 8 13:00:00 mxcpm9kh1101dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u100 - Set Primary nodeid to 1
Mar 8 13:00:01 mxcpm9kh1101dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u119 - Set Primary nodeid to 1
Mar 8 13:00:05 mxcpm9kh1101dc4 Had[26804]: [ID 702911 daemon.notice] VCS ERROR V-16-1-5503 (mxcpm9kh1101dc4) CFSMount:ORREDO_MNT_U03:offline:Umount Faile
d : Mount Point : /u03
Mar 8 13:00:05 mxcpm9kh1101dc4 Had[26804]: [ID 702911 daemon.notice] VCS ERROR V-16-1-5503 (mxcpm9kh1101dc4) CFSMount:ORREDO_MNT_U02:offline:Umount Faile
d : Mount Point : /u02
Mar 8 13:00:05 mxcpm9kh1101dc4 Had[26804]: [ID 702911 daemon.notice] VCS ERROR V-16-1-5503 (mxcpm9kh1101dc4) CFSMount:ORREDO_MNT_U04:offline:Umount Faile
d : Mount Point : /u04
Mar 8 13:00:05 mxcpm9kh1101dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/archives_dg/u05 - Set Primary nodeid to 1
Mar 8 12:59:18 mxcpm9kh1101dc4 last message repeated 3 times
Mar 8 13:00:09 mxcpm9kh1101dc4 sshd[4370]: [ID 800047 auth.notice] Failed none for oracle from 164.2.35.116 port 2646 ssh2
Mar 8 13:00:10 mxcpm9kh1101dc4 sshd[4370]: [ID 800047 auth.info] Accepted keyboard-interactive for oracle from 164.2.35.116 port 2646 ssh2
Mar 8 13:00:14 mxcpm9kh1101dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/redolog_dg/u03 - Set Primary nodeid to 1
Mar 8 13:00:14 mxcpm9kh1101dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/redolog_dg/u02 - Set Primary nodeid to 1
Mar 8 13:00:14 mxcpm9kh1101dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/redolog_dg/u04 - Set Primary nodeid to 1
Mar 8 13:00:17 mxcpm9kh1101dc4 root: [ID 702911 user.error] Oracle CRSD 3532 set to stop
Mar 8 13:00:17 mxcpm9kh1101dc4 root: [ID 702911 user.error] Oracle CRSD 3532 shutdown completed
Mar 8 13:00:17 mxcpm9kh1101dc4 root: [ID 702911 user.error] Oracle EVMD set to stop
Mar 8 13:00:17 mxcpm9kh1101dc4 root: [ID 702911 user.error] Oracle CSSD being stopped
Mar 8 13:00:18 mxcpm9kh1101dc4 root: [ID 702911 user.error] Oracle CSS family monitor shutting down gracefully.
Mar 8 13:00:26 mxcpm9kh1101dc4 root: [ID 702911 user.error] Oracle clsomon shutdown successful.
Mar 8 13:00:27 mxcpm9kh1101dc4 root: [ID 702911 user.error] Oracle CSSD graceful shutdown
Mar 8 13:00:45 mxcpm9kh1101dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/crs_dg/crs_vol - Set Primary nodeid to 1
Mar 8 13:01:02 mxcpm9kh1101dc4 gab: [ID 397130 kern.notice] GAB INFO V-15-1-20032 Port f closed
Mar 8 13:01:05 mxcpm9kh1101dc4 gab: [ID 397130 kern.notice] GAB INFO V-15-1-20032 Port v closed
Mar 8 13:01:05 mxcpm9kh1101dc4 vxvm:vxconfigd: [ID 702911 daemon.notice] V-5-1-7901 CVM_VOLD_STOP command received
Mar 8 13:01:05 mxcpm9kh1101dc4 gab: [ID 397130 kern.notice] GAB INFO V-15-1-20032 Port w closed
Mar 8 13:01:06 mxcpm9kh1101dc4 Had[26804]: [ID 702911 daemon.notice] VCS ERROR V-16-1-1005 (mxcpm9kh1101dc4) CVMCluster:???:monitor:node - state: out of
cluster
Mar 8 13:01:06 mxcpm9kh1101dc4 reason: user initiated stop
Mar 8 13:01:12 mxcpm9kh1101dc4 gab: [ID 316943 kern.notice] GAB INFO V-15-1-20036 Port h gen aeab24 membership 01
Mar 8 13:01:12 mxcpm9kh1101dc4 Had[26804]: [ID 702911 daemon.notice] VCS INFO V-16-1-10077 Received new cluster membership
Mar 8 13:01:12 mxcpm9kh1101dc4 Had[26804]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-10086 System mxcpm9kh1401dc4 (Node '1') is in Regular Membership -
Membership: 0x3
Mar 8 13:01:13 mxcpm9kh1101dc4 gab: [ID 397130 kern.notice] GAB INFO V-15-1-20032 Port h closed
Mar 8 13:01:13 mxcpm9kh1101dc4 mountd[5624]: [ID 598860 daemon.error] svc_create: could not register prog 100231 vers 1 on ticlts
Mar 8 13:01:13 mxcpm9kh1101dc4 mountd[5624]: [ID 598860 daemon.error] svc_create: could not register prog 100231 vers 1 on ticotsord
Mar 8 13:01:13 mxcpm9kh1101dc4 mountd[5624]: [ID 598860 daemon.error] svc_create: could not register prog 100231 vers 1 on ticots
Mar 8 13:01:13 mxcpm9kh1101dc4 mountd[5624]: [ID 882487 daemon.error] unable to create nfsauth service
Mar 8 13:01:14 mxcpm9kh1101dc4 statd[5643]: [ID 514559 daemon.error] svc_tp_create: Could not register prog 100024 vers 1 on udp
Mar 8 13:01:14 mxcpm9kh1101dc4 statd[5643]: [ID 514559 daemon.error] svc_tp_create: Could not register prog 100024 vers 1 on tcp
Mar 8 13:01:14 mxcpm9kh1101dc4 statd[5643]: [ID 514559 daemon.error] svc_tp_create: Could not register prog 100024 vers 1 on ticlts
Mar 8 13:01:14 mxcpm9kh1101dc4 statd[5643]: [ID 514559 daemon.error] svc_tp_create: Could not register prog 100024 vers 1 on ticotsord
Mar 8 13:01:14 mxcpm9kh1101dc4 statd[5643]: [ID 514559 daemon.error] svc_tp_create: Could not register prog 100024 vers 1 on ticots
Mar 8 13:01:14 mxcpm9kh1101dc4 statd[5643]: [ID 447212 daemon.error] statd: unable to create (SM_PROG, SM_VERS) for netpath.
Mar 8 13:01:17 mxcpm9kh1101dc4 Had[5660]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-53021 Diagnostics directory moved to /var/VRTSvcs/diag/had.12680748
77, please check its contents and contact VERITAS Support
Mar 8 13:01:17 mxcpm9kh1101dc4 Had[5663]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-10619 'HAD' starting on: mxcpm9kh1101dc4
Mar 8 13:01:17 mxcpm9kh1101dc4 Had[5663]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-10620 Waiting for local cluster configuration status
Mar 8 13:01:18 mxcpm9kh1101dc4 Had[5663]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-10625 Local cluster configuration valid
Mar 8 13:01:18 mxcpm9kh1101dc4 Had[5663]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-11034 Registering for cluster membership
Mar 8 13:01:18 mxcpm9kh1101dc4 Had[5663]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-11035 Waiting for cluster membership
Mar 8 13:01:23 mxcpm9kh1101dc4 gab: [ID 316943 kern.notice] GAB INFO V-15-1-20036 Port h gen aeab26 membership 01
Mar 8 13:01:23 mxcpm9kh1101dc4 Had[5663]: [ID 702911 daemon.notice] VCS INFO V-16-1-10077 Received new cluster membership
Mar 8 13:01:23 mxcpm9kh1101dc4 Had[5663]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-10086 System mxcpm9kh1101dc4 (Node '0') is in Regular Membership -
Membership: 0x3
Mar 8 13:01:23 mxcpm9kh1101dc4 Had[5663]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-10086 System (Node '1') is in Regular Membership - Membership: 0x3
Mar 8 13:01:23 mxcpm9kh1101dc4 Had[5663]: [ID 702911 daemon.notice] VCS WARNING V-16-1-10072 Cluster waiting for administrative action
Mar 8 13:02:14 mxcpm9kh1101dc4 Had[6104]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-53021 Diagnostics directory moved to /var/VRTSvcs/diag/had.12680749
34, please check its contents and contact VERITAS Support
Mar 8 13:02:14 mxcpm9kh1101dc4 Had[6107]: [ID 702911 daemon.notice] VCS ERROR V-16-1-11009 Exiting: Another copy of VCS may be running
Mar 8 13:02:38 mxcpm9kh1101dc4 gab: [ID 316943 kern.notice] GAB INFO V-15-1-20036 Port h gen aeab27 membership 0
Mar 8 13:02:38 mxcpm9kh1101dc4 gab: [ID 674723 kern.notice] GAB INFO V-15-1-20038 Port h gen aeab27 k_jeopardy ;1
Mar 8 13:02:38 mxcpm9kh1101dc4 gab: [ID 513393 kern.notice] GAB INFO V-15-1-20040 Port h gen aeab27 visible ;1
Mar 8 13:02:38 mxcpm9kh1101dc4 Had[5663]: [ID 702911 daemon.notice] VCS INFO V-16-1-10077 Received new cluster membership
Mar 8 13:02:38 mxcpm9kh1101dc4 Had[5663]: [ID 702911 daemon.notice] VCS ERROR V-16-1-10113 System mxcpm9kh1401dc4 (Node '1') is in DDNA Membership - Memb
ership: 0x1, Visible: 0x0
Mar 8 13:02:39 mxcpm9kh1101dc4 gab: [ID 397130 kern.notice] GAB INFO V-15-1-20032 Port h closed
Mar 8 13:02:49 mxcpm9kh1101dc4 Had[6324]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-53021 Diagnostics directory moved to /var/VRTSvcs/diag/had.12680749
69, please check its contents and contact VERITAS Support
Mar 8 13:02:49 mxcpm9kh1101dc4 Had[6327]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-10619 'HAD' starting on: mxcpm9kh1101dc4
Mar 8 13:02:49 mxcpm9kh1101dc4 Had[6327]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-10620 Waiting for local cluster configuration status
Mar 8 13:02:49 mxcpm9kh1101dc4 Had[6327]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-10625 Local cluster configuration valid
Mar 8 13:02:49 mxcpm9kh1101dc4 Had[6327]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-11034 Registering for cluster membership
Mar 8 13:02:49 mxcpm9kh1101dc4 Had[6327]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-11035 Waiting for cluster membership
Mar 8 13:02:54 mxcpm9kh1101dc4 gab: [ID 316943 kern.notice] GAB INFO V-15-1-20036 Port h gen aeab2c membership 0
Mar 8 13:02:54 mxcpm9kh1101dc4 gab: [ID 674723 kern.notice] GAB INFO V-15-1-20038 Port h gen aeab2c k_jeopardy ;1
Mar 8 13:02:54 mxcpm9kh1101dc4 gab: [ID 513393 kern.notice] GAB INFO V-15-1-20040 Port h gen aeab2c visible ;1
Mar 8 13:02:54 mxcpm9kh1101dc4 Had[6327]: [ID 702911 daemon.notice] VCS INFO V-16-1-10077 Received new cluster membership
Mar 8 13:02:54 mxcpm9kh1101dc4 Had[6327]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-10086 System mxcpm9kh1101dc4 (Node '0') is in Regular Membership -
Membership: 0x1
Mar 8 13:02:54 mxcpm9kh1101dc4 Had[6327]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-10073 Building from local configuration
Mar 8 13:02:55 mxcpm9kh1101dc4 Had[6327]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-10066 Entering RUNNING state
Mar 8 13:02:55 mxcpm9kh1101dc4 Had[6327]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-50311 VCS Engine: running with security OFF
Mar 8 13:02:56 mxcpm9kh1101dc4 Had[6327]: [ID 702911 daemon.notice] VCS ERROR V-16-1-1005 (mxcpm9kh1101dc4) CVMCluster:???:monitor:node - state: out of c
luster
Mar 8 13:02:56 mxcpm9kh1101dc4 reason: user initiated stop
Mar 8 13:03:06 mxcpm9kh1101dc4 gab: [ID 316943 kern.notice] GAB INFO V-15-1-20036 Port v gen aeab27 membership 01
Mar 8 13:03:06 mxcpm9kh1101dc4 vxvm:vxconfigd: [ID 702911 daemon.notice] V-5-1-7900 CVM_VOLD_CONFIG command received
Mar 8 13:03:06 mxcpm9kh1101dc4 vxvm:vxconfigd: [ID 702911 daemon.notice] V-5-1-7899 CVM_VOLD_CHANGE command received
Mar 8 13:03:09 mxcpm9kh1101dc4 gab: [ID 316943 kern.notice] GAB INFO V-15-1-20036 Port h gen aeab2d membership 01
Mar 8 13:03:09 mxcpm9kh1101dc4 Had[6327]: [ID 702911 daemon.notice] VCS INFO V-16-1-10077 Received new cluster membership
Mar 8 13:03:09 mxcpm9kh1101dc4 Had[6327]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-10086 System mxcpm9kh1401dc4 (Node '1') is in Regular Membership -
Membership: 0x3
Mar 8 13:03:11 mxcpm9kh1101dc4 gab: [ID 316943 kern.notice] GAB INFO V-15-1-20036 Port w gen aeab29 membership 01
Mar 8 13:03:13 mxcpm9kh1101dc4 vxvm:vxconfigd: [ID 702911 daemon.notice] V-5-1-12143 CVM_VOLD_JOINOVER command received for node(s) 0
Mar 8 13:03:13 mxcpm9kh1101dc4 vxvm:vxconfigd: [ID 702911 daemon.notice] V-5-1-10994 join completed for node 0
Mar 8 13:03:13 mxcpm9kh1101dc4 vxvm:vxconfigd: [ID 702911 daemon.notice] V-5-1-4123 cluster established successfully
Mar 8 13:03:29 mxcpm9kh1101dc4 gab: [ID 316943 kern.notice] GAB INFO V-15-1-20036 Port f gen aeab2c membership 01
Mar 8 13:03:29 mxcpm9kh1101dc4 vxfs: [ID 779698 kern.notice] GLM recovery : gen aeab2c mbr 3 0 0 0 flags 0
Mar 8 13:03:29 mxcpm9kh1101dc4 vxfs: [ID 702911 kern.notice] NOTICE: msgcnt 9 mesg 125: V-2-125: GLM restart callback, protocol flag 0
Mar 8 13:03:59 mxcpm9kh1101dc4 root: [ID 702911 user.error] Oracle Cluster Ready Services starting by user request.
Mar 8 13:04:19 mxcpm9kh1101dc4 su: [ID 366847 auth.info] 'su oracle' succeeded for root on /dev/???
Mar 8 13:04:21 mxcpm9kh1101dc4 root: [ID 702911 user.error] Cluster Ready Services completed waiting on dependencies.
Mar 8 13:04:21 mxcpm9kh1101dc4 root: [ID 702911 user.error] Oracle CSS Family monitor starting.
Mar 8 13:04:22 mxcpm9kh1101dc4 root: [ID 702911 user.error] Oracle CSS restart. 0, 1
Mar 8 13:04:29 mxcpm9kh1101dc4 root: [ID 702911 user.error] Cluster Ready Services completed waiting on dependencies.
Mar 8 13:04:29 mxcpm9kh1101dc4 last message repeated 1 time
Mar 8 13:04:30 mxcpm9kh1101dc4 root: [ID 702911 user.error] Running CRSD with TZ = Mexico/General
Mar 8 13:04:29 mxcpm9kh1101dc4 last message repeated 5 times
Mar 8 13:06:21 mxcpm9kh1101dc4 su: [ID 366847 auth.info] 'su oracle' succeeded for root on /dev/???
Mar 8 13:06:26 mxcpm9kh1101dc4 last message repeated 7 times
Mar 8 13:15:52 mxcpm9kh1101dc4 AgentFramework[6353]: [ID 702911 daemon.notice] VCS ERROR V-16-1-13067 Thread(5) Agent is calling clean for resource(DANPR
O_ORA) because the resource became OFFLINE unexpectedly, on its own.
Mar 8 13:15:52 mxcpm9kh1101dc4 Had[6327]: [ID 702911 daemon.notice] VCS ERROR V-16-1-13067 (mxcpm9kh1101dc4) Agent is calling clean for resource(DANPRO_O
RA) because the resource became OFFLINE unexpectedly, on its own.
Mar 8 13:15:52 mxcpm9kh1101dc4 su: [ID 366847 auth.info] 'su oracle' succeeded for root on /dev/???
Mar 8 13:15:53 mxcpm9kh1101dc4 AgentFramework[6353]: [ID 702911 daemon.notice] VCS ERROR V-16-1-13068 Thread(5) Resource(DANPRO_ORA) - clean completed su
ccessfully.
Mar 8 13:17:00 mxcpm9kh1101dc4 sshd[24514]: [ID 800047 auth.notice] Failed none for oracle from 164.2.35.118 port 3707 ssh2
Mar 8 13:17:02 mxcpm9kh1101dc4 sshd[24514]: [ID 800047 auth.info] Accepted keyboard-interactive for oracle from 164.2.35.118 port 3707 ssh2
Mar 8 13:46:11 mxcpm9kh1101dc4 Had[6327]: [ID 702911 daemon.notice] VCS ERROR V-16-1-13067 (mxcpm9kh1401dc4) Agent is calling clean for resource(DANPRO_O
RA) because the resource became OFFLINE unexpectedly, on its own.
Mar 8 13:48:54 mxcpm9kh1101dc4 AgentFramework[6353]: [ID 702911 daemon.notice] VCS ERROR V-16-1-13067 Thread(5) Agent is calling clean for resource(DANPR
O_ORA) because the resource became OFFLINE unexpectedly, on its own.
Mar 8 13:48:54 mxcpm9kh1101dc4 Had[6327]: [ID 702911 daemon.notice] VCS ERROR V-16-1-13067 (mxcpm9kh1101dc4) Agent is calling clean for resource(DANPRO_O
RA) because the resource became OFFLINE unexpectedly, on its own.
Mar 8 13:48:54 mxcpm9kh1101dc4 su: [ID 366847 auth.info] 'su oracle' succeeded for root on /dev/???
Mar 8 13:48:55 mxcpm9kh1101dc4 AgentFramework[6353]: [ID 702911 daemon.notice] VCS ERROR V-16-1-13068 Thread(5) Resource(DANPRO_ORA) - clean completed su
ccessfully.
/var/adm/messages extract from node mxcpm9kh1401dc4
Mar 8 12:58:12 mxcpm9kh1401dc4 sshd[29747]: [ID 800047 auth.info] Accepted keyboard-interactive for root from 164.2.45.246 port 49476 ssh2
Mar 8 12:59:13 mxcpm9kh1401dc4 gab: [ID 397130 kern.notice] GAB INFO V-15-1-20032 Port h closed
Mar 8 12:59:13 mxcpm9kh1401dc4 mountd[1406]: [ID 598860 daemon.error] svc_create: could not register prog 100231 vers 1 on ticlts
Mar 8 12:59:13 mxcpm9kh1401dc4 mountd[1406]: [ID 598860 daemon.error] svc_create: could not register prog 100231 vers 1 on ticotsord
Mar 8 12:59:13 mxcpm9kh1401dc4 mountd[1406]: [ID 598860 daemon.error] svc_create: could not register prog 100231 vers 1 on ticots
Mar 8 12:59:13 mxcpm9kh1401dc4 mountd[1406]: [ID 882487 daemon.error] unable to create nfsauth service
Mar 8 12:59:14 mxcpm9kh1401dc4 statd[1441]: [ID 514559 daemon.error] svc_tp_create: Could not register prog 100024 vers 1 on udp
Mar 8 12:59:14 mxcpm9kh1401dc4 statd[1441]: [ID 514559 daemon.error] svc_tp_create: Could not register prog 100024 vers 1 on tcp
Mar 8 12:59:14 mxcpm9kh1401dc4 statd[1441]: [ID 514559 daemon.error] svc_tp_create: Could not register prog 100024 vers 1 on ticlts
Mar 8 12:59:14 mxcpm9kh1401dc4 statd[1441]: [ID 514559 daemon.error] svc_tp_create: Could not register prog 100024 vers 1 on ticotsord
Mar 8 12:59:14 mxcpm9kh1401dc4 statd[1441]: [ID 514559 daemon.error] svc_tp_create: Could not register prog 100024 vers 1 on ticots
Mar 8 12:59:14 mxcpm9kh1401dc4 statd[1441]: [ID 447212 daemon.error] statd: unable to create (SM_PROG, SM_VERS) for netpath.
Mar 8 12:59:31 mxcpm9kh1401dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/faxflow_dg/u09 - Set Primary nodeid to 1
Mar 8 12:59:32 mxcpm9kh1401dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/tandapro_dg/u08 - Set Primary nodeid to 1
Mar 8 12:59:37 mxcpm9kh1401dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/danpro2_dg/danpro2 - Set Primary nodeid to 1
Mar 8 12:59:37 mxcpm9kh1401dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u107 - Set Primary nodeid to 1
Mar 8 12:59:37 mxcpm9kh1401dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u108 - Set Primary nodeid to 1
Mar 8 12:59:37 mxcpm9kh1401dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u106 - Set Primary nodeid to 1
Mar 8 12:59:37 mxcpm9kh1401dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u105 - Set Primary nodeid to 1
Mar 8 12:59:37 mxcpm9kh1401dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u104 - Set Primary nodeid to 1
Mar 8 12:59:37 mxcpm9kh1401dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u103 - Set Primary nodeid to 1
Mar 8 12:59:38 mxcpm9kh1401dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u102 - Set Primary nodeid to 1
Mar 8 12:59:38 mxcpm9kh1401dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u101 - Set Primary nodeid to 1
Mar 8 12:59:38 mxcpm9kh1401dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u110 - Set Primary nodeid to 1
Mar 8 12:59:38 mxcpm9kh1401dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u109 - Set Primary nodeid to 1
Mar 8 12:59:39 mxcpm9kh1401dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u111 - Set Primary nodeid to 1
Mar 8 12:59:39 mxcpm9kh1401dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u113 - Set Primary nodeid to 1
Mar 8 12:59:39 mxcpm9kh1401dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u117 - Set Primary nodeid to 1
Mar 8 12:59:39 mxcpm9kh1401dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u116 - Set Primary nodeid to 1
Mar 8 12:59:39 mxcpm9kh1401dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u114 - Set Primary nodeid to 1
Mar 8 12:59:39 mxcpm9kh1401dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u118 - Set Primary nodeid to 1
Mar 8 12:59:40 mxcpm9kh1401dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u115 - Set Primary nodeid to 1
Mar 8 12:59:40 mxcpm9kh1401dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u112 - Set Primary nodeid to 1
Mar 8 12:59:41 mxcpm9kh1401dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/crystwb_dg/u07 - Set Primary nodeid to 1
Mar 8 13:00:00 mxcpm9kh1401dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u100 - Set Primary nodeid to 1
Mar 8 13:00:01 mxcpm9kh1401dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/oraseg1_dg/u119 - Set Primary nodeid to 1
Mar 8 13:00:05 mxcpm9kh1401dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/archives_dg/u05 - Set Primary nodeid to 1
Mar 8 13:00:14 mxcpm9kh1401dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/redolog_dg/u03 - Set Primary nodeid to 1
Mar 8 13:00:14 mxcpm9kh1401dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/redolog_dg/u02 - Set Primary nodeid to 1
Mar 8 13:00:14 mxcpm9kh1401dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/redolog_dg/u04 - Set Primary nodeid to 1
Mar 8 13:00:45 mxcpm9kh1401dc4 vxfs: [ID 355872 kern.notice] Phase 9 - /dev/vx/dsk/crs_dg/crs_vol - Set Primary nodeid to 1
Mar 8 13:01:06 mxcpm9kh1401dc4 Had[2695]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-53021 Diagnostics directory moved to /var/VRTSvcs/diag/had.12680748
66, please check its contents and contact VERITAS Support
Mar 8 13:01:06 mxcpm9kh1401dc4 Had[2698]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-10619 'HAD' starting on: mxcpm9kh1401dc4
Mar 8 13:01:06 mxcpm9kh1401dc4 Had[2698]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-10620 Waiting for local cluster configuration status
Mar 8 13:01:06 mxcpm9kh1401dc4 gab: [ID 316943 kern.notice] GAB INFO V-15-1-20036 Port f gen aeab2b membership ;1
Mar 8 13:01:06 mxcpm9kh1401dc4 gab: [ID 674723 kern.notice] GAB INFO V-15-1-20038 Port f gen aeab2b k_jeopardy 0
Mar 8 13:01:06 mxcpm9kh1401dc4 gab: [ID 513393 kern.notice] GAB INFO V-15-1-20040 Port f gen aeab2b visible 0
Mar 8 13:01:06 mxcpm9kh1401dc4 vxfs: [ID 785551 kern.notice] I/O fencing is ON
Mar 8 13:01:07 mxcpm9kh1401dc4 Had[2698]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-10625 Local cluster configuration valid
Mar 8 13:01:07 mxcpm9kh1401dc4 Had[2698]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-11034 Registering for cluster membership
Mar 8 13:01:07 mxcpm9kh1401dc4 Had[2698]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-11035 Waiting for cluster membership
Mar 8 13:01:12 mxcpm9kh1401dc4 gab: [ID 316943 kern.notice] GAB INFO V-15-1-20036 Port h gen aeab24 membership 01
Mar 8 13:01:12 mxcpm9kh1401dc4 Had[2698]: [ID 702911 daemon.notice] VCS INFO V-16-1-10077 Received new cluster membership
Mar 8 13:01:12 mxcpm9kh1401dc4 Had[2698]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-10086 System (Node '0') is in Regular Membership - Membership: 0x3
Mar 8 13:01:12 mxcpm9kh1401dc4 Had[2698]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-10086 System mxcpm9kh1401dc4 (Node '1') is in Regular Membership -
Membership: 0x3
Mar 8 13:01:12 mxcpm9kh1401dc4 Had[2698]: [ID 702911 daemon.notice] VCS ERROR V-16-1-50129 Operation 'hares -modify' rejected as the node is in CURRENT_D
ISCOVER_WAIT state
Mar 8 13:01:12 mxcpm9kh1401dc4 last message repeated 1 time
Mar 8 13:01:12 mxcpm9kh1401dc4 Had[2698]: [ID 702911 daemon.notice] VCS WARNING V-16-1-10072 Cluster waiting for administrative action
Mar 8 13:01:17 mxcpm9kh1401dc4 gab: [ID 316943 kern.notice] GAB INFO V-15-1-20036 Port h gen aeab25 membership ;1
Mar 8 13:01:17 mxcpm9kh1401dc4 gab: [ID 674723 kern.notice] GAB INFO V-15-1-20038 Port h gen aeab25 k_jeopardy 0
Mar 8 13:01:17 mxcpm9kh1401dc4 gab: [ID 513393 kern.notice] GAB INFO V-15-1-20040 Port h gen aeab25 visible 0
Mar 8 13:01:17 mxcpm9kh1401dc4 Had[2698]: [ID 702911 daemon.notice] VCS INFO V-16-1-10077 Received new cluster membership
Mar 8 13:01:17 mxcpm9kh1401dc4 Had[2698]: [ID 702911 daemon.notice] VCS ERROR V-16-1-10113 System mxcpm9kh1101dc4 (Node '0') is in DDNA Membership - Memb
ership: 0x2, Visible: 0x0
Mar 8 13:01:23 mxcpm9kh1401dc4 gab: [ID 316943 kern.notice] GAB INFO V-15-1-20036 Port h gen aeab26 membership 01
Mar 8 13:01:23 mxcpm9kh1401dc4 Had[2698]: [ID 702911 daemon.notice] VCS INFO V-16-1-10077 Received new cluster membership
Mar 8 13:01:23 mxcpm9kh1401dc4 Had[2698]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-10086 System (Node '0') is in Regular Membership - Membership: 0x3
Mar 8 13:01:23 mxcpm9kh1401dc4 vxvm:vxconfigd: [ID 702911 daemon.notice] V-5-1-8062 master: not a cluster startup
Mar 8 13:01:23 mxcpm9kh1401dc4 vxvm:vxconfigd: [ID 702911 daemon.notice] V-5-1-10994 join completed for node 1
Mar 8 13:01:23 mxcpm9kh1401dc4 vxvm:vxconfigd: [ID 702911 daemon.notice] V-5-1-4123 cluster established successfully
Mar 8 13:02:09 mxcpm9kh1401dc4 Had[3406]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-53021 Diagnostics directory moved to /var/VRTSvcs/diag/had.12680749
29, please check its contents and contact VERITAS Support
Mar 8 13:02:09 mxcpm9kh1401dc4 Had[3414]: [ID 702911 daemon.notice] VCS ERROR V-16-1-11009 Exiting: Another copy of VCS may be running
Mar 8 13:02:18 mxcpm9kh1401dc4 Had[3554]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-53021 Diagnostics directory moved to /var/VRTSvcs/diag/had.12680749
38, please check its contents and contact VERITAS Support
Mar 8 13:02:18 mxcpm9kh1401dc4 Had[3557]: [ID 702911 daemon.notice] VCS ERROR V-16-1-11009 Exiting: Another copy of VCS may be running
Mar 8 13:02:33 mxcpm9kh1401dc4 gab: [ID 397130 kern.notice] GAB INFO V-15-1-20032 Port h closed
Mar 8 13:03:03 mxcpm9kh1401dc4 Had[4062]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-53021 Diagnostics directory moved to /var/VRTSvcs/diag/had.12680749
83, please check its contents and contact VERITAS Support
Mar 8 13:03:03 mxcpm9kh1401dc4 Had[4065]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-10619 'HAD' starting on: mxcpm9kh1401dc4
Mar 8 13:03:03 mxcpm9kh1401dc4 Had[4065]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-10620 Waiting for local cluster configuration status
Mar 8 13:03:03 mxcpm9kh1401dc4 Had[4065]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-10625 Local cluster configuration valid
Mar 8 13:03:03 mxcpm9kh1401dc4 Had[4065]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-11034 Registering for cluster membership
Mar 8 13:03:03 mxcpm9kh1401dc4 Had[4065]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-11035 Waiting for cluster membership
Mar 8 13:03:06 mxcpm9kh1401dc4 gab: [ID 316943 kern.notice] GAB INFO V-15-1-20036 Port v gen aeab27 membership 01
Mar 8 13:03:06 mxcpm9kh1401dc4 vxvm:vxconfigd: [ID 702911 daemon.notice] V-5-1-7899 CVM_VOLD_CHANGE command received
Mar 8 13:03:06 mxcpm9kh1401dc4 vxvm:vxconfigd: [ID 702911 daemon.notice] V-5-1-8062 master: not a cluster startup
Mar 8 13:03:09 mxcpm9kh1401dc4 gab: [ID 316943 kern.notice] GAB INFO V-15-1-20036 Port h gen aeab2d membership 01
Mar 8 13:03:09 mxcpm9kh1401dc4 Had[4065]: [ID 702911 daemon.notice] VCS INFO V-16-1-10077 Received new cluster membership
Mar 8 13:03:09 mxcpm9kh1401dc4 Had[4065]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-10086 System (Node '0') is in Regular Membership - Membership: 0x3
Mar 8 13:03:09 mxcpm9kh1401dc4 Had[4065]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-10086 System mxcpm9kh1401dc4 (Node '1') is in Regular Membership -
Membership: 0x3
Mar 8 13:03:09 mxcpm9kh1401dc4 Had[4065]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-10075 Building from remote system
Mar 8 13:03:10 mxcpm9kh1401dc4 Had[4065]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-10066 Entering RUNNING state
Mar 8 13:03:10 mxcpm9kh1401dc4 Had[4065]: [ID 702911 daemon.notice] VCS NOTICE V-16-1-50311 VCS Engine: running with security OFF
Mar 8 13:03:11 mxcpm9kh1401dc4 gab: [ID 316943 kern.notice] GAB INFO V-15-1-20036 Port w gen aeab29 membership 01
Mar 8 13:03:13 mxcpm9kh1401dc4 vxvm:vxconfigd: [ID 702911 daemon.notice] V-5-1-12143 CVM_VOLD_JOINOVER command received for node(s) 0
Mar 8 13:03:13 mxcpm9kh1401dc4 vxvm:vxconfigd: [ID 702911 daemon.notice] V-5-1-10994 join completed for node 1
Mar 8 13:03:13 mxcpm9kh1401dc4 vxvm:vxconfigd: [ID 702911 daemon.notice] V-5-1-4123 cluster established successfully
Mar 8 13:03:29 mxcpm9kh1401dc4 gab: [ID 316943 kern.notice] GAB INFO V-15-1-20036 Port f gen aeab2c membership 01
Mar 8 13:03:29 mxcpm9kh1401dc4 vxfs: [ID 779698 kern.notice] GLM recovery : gen aeab2c mbr 3 0 0 0 flags 0
Mar 8 13:03:29 mxcpm9kh1401dc4 vxfs: [ID 702911 kern.notice] NOTICE: msgcnt 6 mesg 125: V-2-125: GLM restart callback, protocol flag 0