Listener logs : tail -1000

allaboutunix · ‎08-31-2015

Team,

During a database activity, we did not freeze the cluster due to some reason,the listener went down due to the activity however database team has confirmed that DB is up and online.

Initially, when i tried to check the faulted resource it was not showing.But, when I tried to clear and online the listener resource, it went to Waiting_to_online and then it failed.

Could you please statehow we can resolve and make the listener online.

Below are the information,

A turks RUNNING 0
A madagascar RUNNING 0

-- GROUP STATE
-- Group System Probed AutoDisabled State

B ClusterService turks Y N OFFLINE
B ClusterService madagascar Y N ONLINE
B DB_TRS_PRD turks Y N OFFLINE
B DB_TRS_PRD madagascar Y N PARTIAL

-- RESOURCES FAILED
-- Group Type Resource System

D DB_TRS_PRD Netlsnr trsprd_LSNR madagascar

================================================================

Main.cf O/P -

/etc/VRTSvcs/conf/config/main.cf
include "OracleASMTypes.cf"
include "types.cf"
include "Db2udbTypes.cf"
include "OracleTypes.cf"
include "SybaseTypes.cf"

cluster TRS-PRD-MG (
UserNames = { admin = cmnFmhMjnInnLvnHmk,
rengasak = JiiMjeHljEikEdfDef,
g023154 = hifEfiEciGjiHhiP,
sparisd = aPPsMSnONjPNoNMlMR,
guptan2 = JljQifKmlNltHhkKliLi,
g035006 = hqqWrnQqqSqyPlqVns }
Administrators = { admin, rengasav, g023154, sparisd, guptau5, g035006 }
)

system turks (
)

system madagascar (
)

group ClusterService (
SystemList = { turks = 0, madagascar = 1 }
AutoStartList = { turks, madagascar }
OnlineRetryLimit = 3
OnlineRetryInterval = 120
)

NIC csgnic (
Device = en2
NetworkHosts = { "132.189.210.1" }
)

NotifierMngr ntfr (
SmtpServer = "mapintmail@infi.com"
SmtpRecipients = { "systemsunix@infi.com" = Error }
)

ntfr requires csgnic

// resource dependency tree
//
// group ClusterService
// {
// NotifierMngr ntfr
// {
// NIC csgnic
// }
// }

group DB_TRS_PRD (
SystemList = { madagascar = 0, turks = 1 }
Frozen = 1
)

IP trsprd_ora_vip (
Device = en2
Address = "132.189.210.138"
NetMask = "255.255.255.0"
)

LVMVG trsprd_ora_lvmvg (
VolumeGroup = db_trsprd
MajorNumber = 40
)

Mount db_trsprd_arch (
MountPoint = "/db_trsprd/arch"
BlockDevice = "/dev/trsprd_arch"
FSType = jfs2
FsckOpt = "-y"
)

Mount db_trsprd_backup (
MountPoint = "/db_trsprd/backup"
BlockDevice = "/dev/trsprd_backup"
FSType = jfs2
FsckOpt = "-y"
)

Mount db_trsprd_data1 (
MountPoint = "/db_trsprd/data1"
BlockDevice = "/dev/trsprd_data1"
FSType = jfs2
FsckOpt = "-y"
)

Mount db_trsprd_data2 (
MountPoint = "/db_trsprd/data2"
BlockDevice = "/dev/trsprd_data2"
FSType = jfs2
FsckOpt = "-y"
)

Mount db_trsprd_data3 (
MountPoint = "/db_trsprd/data3"
BlockDevice = "/dev/trsprd_data3"
FSType = jfs2
FsckOpt = "-y"
)

Mount db_trsprd_dba (
MountPoint = "/db_trsprd/dba"
BlockDevice = "/dev/trsprd_dba"
FSType = jfs2
FsckOpt = "-y"
)

Mount db_trsprd_oracle (
MountPoint = "/db_trsprd/oracle"
BlockDevice = "/dev/trsprd_oracle"
FSType = jfs2
FsckOpt = "-y"
)

Mount db_trsprd_redo1 (
MountPoint = "/db_trsprd/redo1"
BlockDevice = "/dev/trsprd_redo1"
FSType = jfs2
FsckOpt = "-y"
)

Mount db_trsprd_redo2 (
MountPoint = "/db_trsprd/redo2"
BlockDevice = "/dev/trsprd_redo2"
FSType = jfs2
FsckOpt = "-y"
)

Mount db_trsprd_temp (
MountPoint = "/db_trsprd/temp"
BlockDevice = "/dev/trsprd_temp"
FSType = jfs2
FsckOpt = "-y"
)

Netlsnr trsprd_LSNR (
Critical = 0
Owner = oracle
Home = "/db_trsprd/oracle/product/11.2.0.2"
TnsAdmin = "/var/opt/oracle"
Listener = LISTENER_TRSPRD
EnvFile = "/oracle/home/.profile"
)

Oracle trsprd_ORA (
Critical = 0
Sid = trsprd
Owner = oracle
Home = "/db_trsprd/oracle/product/11.2.0.2"
EnvFile = "/oracle/home/.profile"
)

Proxy trsprd_NIC (
TargetResName = csgnic
)

db_trsprd_arch requires trsprd_ora_lvmvg
db_trsprd_backup requires trsprd_ora_lvmvg
db_trsprd_data1 requires trsprd_ora_lvmvg
db_trsprd_data2 requires trsprd_ora_lvmvg
db_trsprd_data3 requires trsprd_ora_lvmvg
db_trsprd_dba requires trsprd_ora_lvmvg
db_trsprd_oracle requires trsprd_ora_lvmvg
db_trsprd_redo1 requires trsprd_ora_lvmvg
db_trsprd_redo2 requires trsprd_ora_lvmvg
db_trsprd_temp requires trsprd_ora_lvmvg
trsprd_LSNR requires trsprd_ORA
trsprd_LSNR requires trsprd_ora_vip
trsprd_ORA requires db_trsprd_arch
trsprd_ORA requires db_trsprd_backup
trsprd_ORA requires db_trsprd_data1
trsprd_ORA requires db_trsprd_data2
trsprd_ORA requires db_trsprd_data3
trsprd_ORA requires db_trsprd_dba
trsprd_ORA requires db_trsprd_oracle
trsprd_ORA requires db_trsprd_redo1
trsprd_ORA requires db_trsprd_redo2
trsprd_ORA requires db_trsprd_temp
trsprd_ora_vip requires trsprd_NIC

// resource dependency tree
//
// group DB_TRS_PRD
// {
// Netlsnr trsprd_LSNR
// {
// IP trsprd_ora_vip
// {
// Proxy trsprd_NIC
// }
// Oracle trsprd_ORA
// {
// Mount db_trsprd_temp
// {
// LVMVG trsprd_ora_lvmvg
// }
// Mount db_trsprd_backup
// {
// LVMVG trsprd_ora_lvmvg
// }
// Mount db_trsprd_arch
// {
// LVMVG trsprd_ora_lvmvg
// }
// Mount db_trsprd_redo2
// {
// LVMVG trsprd_ora_lvmvg
// }
// Mount db_trsprd_redo1
// {
// LVMVG trsprd_ora_lvmvg
// }
// Mount db_trsprd_oracle
// {
// LVMVG trsprd_ora_lvmvg
// }
// Mount db_trsprd_dba
// {
// LVMVG trsprd_ora_lvmvg
// }
// Mount db_trsprd_data2
// {
// LVMVG trsprd_ora_lvmvg
// }
// Mount db_trsprd_data1
// {
// LVMVG trsprd_ora_lvmvg
// }
// Mount db_trsprd_data3
// {
// LVMVG trsprd_ora_lvmvg
// }
// }
// }
// }

=======================================================

sudhir_h · ‎08-31-2015

You would need to identify the reason for the fault of Netlsnr resource. Enable debug logging for Netlsnr resource and then try to online the resource.

# haconfig -makerw

# hatype -modify Netlsnr 1 2 3 4 5

Check the /var/VRTSvcs/log/Netlsnr_A.log and /var/VRTSvcs/log/engine_A.log log files.

Attach the log if possible.

Regards,

Sudhir

allaboutunix · ‎09-01-2015

Listener logs :

tail -1000 /var/VRTSvcs/log/Netlsnr_A.log
#
# Log Name: Netlsnr
# System: madagascar
# SysInfo: Aix:madagascar,6,1,00F6CEA14C00
# Created: 2014/03/20 16:40:16
#

2014/03/20 16:40:16 VCS INFO V-16-20002-240 Netlsnr:Netlsnr:imf_init:Successfully initialized the driver with reaper:(VCSNetlsnrAgent), pid:(14286978)
2014/03/20 16:40:16 VCS INFO V-16-2-13805 Thread(1) (imf_init) entry point completed with return status (0)
2014/03/20 16:42:59 VCS WARNING V-16-2-13112 Thread(1) Unknown resource name trsodsp_LSNR
2014/03/20 16:43:22 VCS NOTICE V-16-20002-210 Netlsnr:trsodsp_LSNR:monitor:Setting cookie for proc = /db_trsodsp/oracle/product/11.2.0.2/bin/tnslsnr LISTENER_TRSODSP -inherit, PID = 6160414
2014/03/21 10:07:48 VCS ERROR V-16-2-13066 Thread(1030) Agent is calling clean for resource(trsprd_LSNR) because the resource is not up even after online completed.
2014/03/21 10:07:50 VCS ERROR V-16-2-13068 Thread(1030) Resource(trsprd_LSNR) - clean completed successfully.
2014/03/21 10:07:50 VCS ERROR V-16-2-13071 Thread(1030) Resource(trsprd_LSNR): reached OnlineRetryLimit(0).
2014/03/21 10:11:51 VCS ERROR V-16-2-13066 Thread(1030) Agent is calling clean for resource(trsprd_LSNR) because the resource is not up even after online completed.
2014/03/21 10:11:53 VCS ERROR V-16-2-13068 Thread(1030) Resource(trsprd_LSNR) - clean completed successfully.
2014/03/21 10:11:53 VCS ERROR V-16-2-13071 Thread(1030) Resource(trsprd_LSNR): reached OnlineRetryLimit(0).
2014/03/21 10:22:05 VCS ERROR V-16-2-13066 Thread(1030) Agent is calling clean for resource(trsprd_LSNR) because the resource is not up even after online completed.
2014/03/21 10:22:07 VCS ERROR V-16-2-13068 Thread(1030) Resource(trsprd_LSNR) - clean completed successfully.
2014/03/21 10:22:07 VCS ERROR V-16-2-13071 Thread(1030) Resource(trsprd_LSNR): reached OnlineRetryLimit(0).
2014/03/21 10:35:51 VCS NOTICE V-16-20002-210 Netlsnr:trsprd_LSNR:monitor:Setting cookie for proc = /db_trsprd/oracle/product/11.2.0.2/bin/tnslsnr LISTENER_TRSPRD -inherit, PID = 26279980
2014/03/21 10:37:27 VCS NOTICE V-16-20002-210 Netlsnr:trsodsp_LSNR:monitor:Setting cookie for proc = /db_trsodsp/oracle/product/11.2.0.2/bin/tnslsnr LISTENER_TRSODSP -inherit, PID = 23134304
2014/04/05 09:43:33 VCS NOTICE V-16-20002-210 Netlsnr:trsodsp_LSNR:monitor:Setting cookie for proc = /db_trsodsp/oracle/product/11.2.0.2/bin/tnslsnr LISTENER_TRSODSP -inherit, PID = 22872238
2014/04/05 12:15:05 VCS WARNING V-16-20002-205 Netlsnr:trsodsp_LSNR:monitor:Process tnslsnr restarted
2014/04/05 11:36:49 VCS INFO V-16-20002-240 Netlsnr:Netlsnr:imf_init:Successfully initialized the driver with reaper:(VCSNetlsnrAgent), pid:(4980806)
2014/04/05 11:36:49 VCS INFO V-16-2-13805 Thread(1) (imf_init) entry point completed with return status (0)
2014/04/05 11:54:43 VCS NOTICE V-16-20002-210 Netlsnr:trsodsp_LSNR:monitor:Setting cookie for proc = /db_trsodsp/oracle/product/11.2.0.2/bin/tnslsnr LISTENER_TRSODSP -inherit, PID = 3866828
2014/04/26 21:17:23 VCS INFO V-16-20002-240 Netlsnr:Netlsnr:imf_init:Successfully initialized the driver with reaper:(VCSNetlsnrAgent), pid:(7012566)
2014/04/26 21:17:23 VCS INFO V-16-2-13805 Thread(1) (imf_init) entry point completed with return status (0)
2014/04/26 21:35:51 VCS NOTICE V-16-20002-210 Netlsnr:trsodsp_LSNR:monitor:Setting cookie for proc = /db_trsodsp/oracle/product/11.2.0.2/bin/tnslsnr LISTENER_TRSODSP -inherit, PID = 8912986
2014/07/19 20:51:26 VCS INFO V-16-20002-240 Netlsnr:Netlsnr:imf_init:Successfully initialized the driver with reaper:(VCSNetlsnrAgent), pid:(9633830)
2014/07/19 20:51:26 VCS INFO V-16-2-13805 Thread(1) (imf_init) entry point completed with return status (0)
2014/07/19 21:02:32 VCS NOTICE V-16-20002-210 Netlsnr:trsodsp_LSNR:monitor:Setting cookie for proc = /db_trsodsp/oracle/product/11.2.0.2/bin/tnslsnr LISTENER_TRSODSP -inherit, PID = 6947064
Thread(772) Error (2) received while fetching psinfo for pid (15728862)2014/09/02 18:51:38 VCS NOTICE V-16-20002-210 Netlsnr:trsprd_LSNR:monitor:Setting cookie for proc = /db_trsprd/oracle/product/11.2.0.2/bin/tnslsnr LISTENER_TRSPRD -inherit, PID = 8192106
2014/09/02 19:16:15 VCS WARNING V-16-20002-205 Netlsnr:trsodsp_LSNR:monitor:Process tnslsnr restarted
2015/01/26 11:07:30 VCS NOTICE V-16-20002-210 Netlsnr:trsprd_LSNR:monitor:Setting cookie for proc = /db_trsprd/oracle/product/11.2.0.2/bin/tnslsnr LISTENER_TRSPRD -inherit, PID = 24838262
2015/02/21 19:08:55 VCS NOTICE V-16-20002-210 Netlsnr:trsodsp_LSNR:monitor:Setting cookie for proc = /db_trsodsp/oracle/product/11.2.0.2/bin/tnslsnr LISTENER_TRSODSP -inherit, PID = 32047200
2015/02/21 21:14:56 VCS WARNING V-16-20002-205 Netlsnr:trsodsp_LSNR:monitor:Process tnslsnr restarted
2015/04/11 19:42:54 VCS WARNING V-16-20002-205 Netlsnr:trsprd_LSNR:monitor:Process tnslsnr restarted
2015/04/11 21:00:28 VCS NOTICE V-16-20002-210 Netlsnr:trsprd_LSNR:monitor:Setting cookie for proc = /db_trsprd/oracle/product/11.2.0.2/bin/tnslsnr LISTENER_TRSPRD -inherit, PID = 17694836
2015/06/21 23:46:08 VCS NOTICE V-16-20002-210 Netlsnr:trsprd_LSNR:monitor:Setting cookie for proc = /db_trsprd/oracle/product/11.2.0.2/bin/tnslsnr LISTENER_TRSPRD -inherit, PID = 63373516
2015/07/17 06:17:27 VCS WARNING V-16-2-13108 Thread(1) Engine reported error(e02)
2015/07/17 06:17:27 VCS WARNING V-16-2-13109 Thread(1) Engine reported error; error info is (Resource does not exist: trsodsp_LSNR)
2015/08/30 22:54:39 VCS ERROR V-16-2-13066 Thread(772) Agent is calling clean for resource(trsprd_LSNR) because the resource is not up even after online completed.
2015/08/30 22:54:40 VCS ERROR V-16-2-13068 Thread(772) Resource(trsprd_LSNR) - clean completed successfully.
2015/08/30 22:54:40 VCS ERROR V-16-2-13071 Thread(772) Resource(trsprd_LSNR): reached OnlineRetryLimit(0).
2015/08/30 23:02:09 VCS ERROR V-16-2-13066 Thread(772) Agent is calling clean for resource(trsprd_LSNR) because the resource is not up even after online completed.
2015/08/30 23:02:10 VCS ERROR V-16-2-13068 Thread(772) Resource(trsprd_LSNR) - clean completed successfully.
2015/08/30 23:02:10 VCS ERROR V-16-2-13071 Thread(772) Resource(trsprd_LSNR): reached OnlineRetryLimit(0).
2015/08/30 23:05:48 VCS ERROR V-16-2-13066 Thread(772) Agent is calling clean for resource(trsprd_LSNR) because the resource is not up even after online completed.
2015/08/30 23:05:49 VCS ERROR V-16-2-13068 Thread(772) Resource(trsprd_LSNR) - clean completed successfully.
2015/08/30 23:05:50 VCS ERROR V-16-2-13071 Thread(772) Resource(trsprd_LSNR): reached OnlineRetryLimit(0).
2015/08/31 05:53:40 VCS ERROR V-16-2-13066 Thread(772) Agent is calling clean for resource(trsprd_LSNR) because the resource is not up even after online completed.
2015/08/31 05:53:41 VCS ERROR V-16-2-13068 Thread(772) Resource(trsprd_LSNR) - clean completed successfully.
2015/08/31 05:53:41 VCS ERROR V-16-2-13071 Thread(772) Resource(trsprd_LSNR): reached OnlineRetryLimit(0).

==============================================================================

engineA.logs :

2015/08/31 05:51:30 VCS INFO V-16-1-10307 Resource trsprd_LSNR (Owner: Unspecified, Group: DB_TRS_PRD) is offline on madagascar (Not initiated by VCS)
2015/08/31 05:51:39 VCS NOTICE V-16-1-10233 Clearing Restart attribute for group DB_TRS_PRD on all nodes
2015/08/31 05:51:39 VCS NOTICE V-16-1-10301 Initiating Online of Resource trsprd_LSNR (Owner: Unspecified, Group: DB_TRS_PRD) on System madagascar
2015/08/31 05:51:39 VCS NOTICE V-16-20002-68 (madagascar) Netlsnr:trsprd_LSNR:online:lsnrctl returned the following output: +----------------------------------
----------------------------------+
2015/08/31 05:51:40 VCS INFO V-16-2-13716 (madagascar) Resource(trsprd_LSNR): Output of the completed operation (online)
2015/08/31 05:53:40 VCS ERROR V-16-2-13066 (madagascar) Agent is calling clean for resource(trsprd_LSNR) because the resource is not up even after online compl
eted.
2015/08/31 05:53:41 VCS CRITICAL V-16-20002-41 (madagascar) Netlsnr:trsprd_LSNR:clean:Listener process LISTENER_TRSPRD not running
2015/08/31 05:53:41 VCS INFO V-16-2-13068 (madagascar) Resource(trsprd_LSNR) - clean completed successfully.
2015/08/31 05:53:41 VCS INFO V-16-2-13071 (madagascar) Resource(trsprd_LSNR): reached OnlineRetryLimit(0).
2015/08/31 05:53:41 VCS ERROR V-16-1-54031 Resource trsprd_LSNR (Owner: Unspecified, Group: DB_TRS_PRD) is FAULTED on sys madagascar
2015/08/31 05:53:42 VCS INFO V-16-6-15015 (madagascar) hatrigger:/opt/VRTSvcs/bin/triggers/resfault is not a trigger scripts directory or can not be executed

sudhir_h · ‎09-01-2015

The engine log suggests that Listener was not able to run successfully.

2015/08/31 05:51:39 VCS NOTICE V-16-20002-68 (madagascar) Netlsnr:trsprd_LSNR:online:lsnrctl returned the following output: +----------------------------------
----------------------------------+

Usually when the Netlsnr online runs successfully, you will see output similar to below.

2015/08/07 13:11:38 VCS NOTICE V-16-20002-68 Netlsnr:listsener_res:online:lsnrctl returned the following output: +--------------------------------------------------------------------+
LD_LIBRARY_PATH - /usr/lib:

LSNRCTL for Linux: Version 11.2.0.4.0 - Production on 07-AUG-2015 13:11:38

Starting /oracle/orahome/bin/tnslsnr: please wait...

TNSLSNR for Linux: Version 11.2.0.4.0 - Production
System parameter file is /oracle/orahome/network/admin/listener.ora
Log messages written to /oracle/diag/tnslsnr/vcslx325/listener/alert/log.xml
Listening on: (DESCRIPTION=(ADDRESS=(PROTOCOL=tcp)(HOST=XXX.xyz.com)(PORT=1521)))
Listening on: (DESCRIPTION=(ADDRESS=(PROTOCOL=ipc)(KEY=EXTPROC1521)))

Connecting to (DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=XXX.xyz.com)(PORT=1521)))
STATUS of the LISTENER
------------------------
Alias                     LISTENER
Version                   TNSLSNR for Linux: Version 11.2.0.4.0 - Production
Start Date                07-AUG-2015 13:11:38
Uptime                    0 days 0 hr. 0 min. 0 sec
Trace Level               off
Security                  ON: Local OS Authentication
SNMP                      OFF
Listener Parameter File   /oracle/orahome/network/admin/listener.ora
Listener Log File         /oracle/diag/tnslsnr/XXX/listener/alert/log.xml
Listening Endpoints Summary...
(DESCRIPTION=(ADDRESS=(PROTOCOL=tcp)(HOST=XXX.xyz.com)(PORT=1521)))
(DESCRIPTION=(ADDRESS=(PROTOCOL=ipc)(KEY=EXTPROC1521)))
The listener supports no services
The command completed successfully

Can you enable the debug level and try performing the online operation?

Also are you able to bring the listener up manually outside VCS control and does VCS report the proper state?

# su - oracle

# source /oracle/home/.profile

# lsnrctl start LISTENER_TRSPRD

# lsnrctl status LISTENER_TRSPRD

Regards

VOX

Listener failed