Forum Discussion

allaboutunix's avatar
10 years ago

Listener failed

Team,

 

During a database activity, we  did not freeze the cluster due to some reason,the listener went down due to the activity however database team has confirmed that DB is up and online.

Initially, when i tried to check  the faulted resource it was not showing.But, when I tried to clear  and online the listener resource, it went to Waiting_to_online and then it failed.

Could you please statehow we can resolve and make the listener online.

Below are the information,

 


A   turks           RUNNING              0
A  madagascar           RUNNING              0

-- GROUP STATE
-- Group           System               Probed     AutoDisabled    State

B  ClusterService    turks            Y          N               OFFLINE
B  ClusterService    madagascar            Y          N               ONLINE
B  DB_TRS_PRD        turks            Y          N               OFFLINE
B  DB_TRS_PRD       madagascar            Y          N               PARTIAL

-- RESOURCES FAILED
-- Group           Type                 Resource             System

D  DB_TRS_PRD      Netlsnr              trsprd_LSNR          madagascar

================================================================

Main.cf O/P -

/etc/VRTSvcs/conf/config/main.cf
include "OracleASMTypes.cf"
include "types.cf"
include "Db2udbTypes.cf"
include "OracleTypes.cf"
include "SybaseTypes.cf"

cluster TRS-PRD-MG (
        UserNames = { admin = cmnFmhMjnInnLvnHmk,
                 rengasak = JiiMjeHljEikEdfDef,
                 g023154 = hifEfiEciGjiHhiP,
                 sparisd = aPPsMSnONjPNoNMlMR,
                 guptan2 = JljQifKmlNltHhkKliLi,
                 g035006 = hqqWrnQqqSqyPlqVns }
        Administrators = { admin, rengasav, g023154, sparisd, guptau5, g035006 }
        )

system turks (
        )

system madagascar (
        )

group ClusterService (
        SystemList = { turks = 0, madagascar = 1 }
        AutoStartList = { turks, madagascar }
        OnlineRetryLimit = 3
        OnlineRetryInterval = 120
        )

        NIC csgnic (
                Device = en2
                NetworkHosts = { "132.189.210.1" }
                )

        NotifierMngr ntfr (
                SmtpServer = "mapintmail@infi.com"
                SmtpRecipients = { "systemsunix@infi.com" = Error }
                )

        ntfr requires csgnic


        // resource dependency tree
        //
        //      group ClusterService
        //      {
        //      NotifierMngr ntfr
        //          {
        //          NIC csgnic
        //          }
        //      }


group DB_TRS_PRD (
        SystemList = { madagascar = 0, turks = 1 }
        Frozen = 1
        )

        IP trsprd_ora_vip (
                Device = en2
                Address = "132.189.210.138"
                NetMask = "255.255.255.0"
                )

        LVMVG trsprd_ora_lvmvg (
                VolumeGroup = db_trsprd
                MajorNumber = 40
                )

        Mount db_trsprd_arch (
                MountPoint = "/db_trsprd/arch"
                BlockDevice = "/dev/trsprd_arch"
                FSType = jfs2
                FsckOpt = "-y"
                )

        Mount db_trsprd_backup (
                MountPoint = "/db_trsprd/backup"
                BlockDevice = "/dev/trsprd_backup"
                FSType = jfs2
                FsckOpt = "-y"
                )

        Mount db_trsprd_data1 (
                MountPoint = "/db_trsprd/data1"
                BlockDevice = "/dev/trsprd_data1"
                FSType = jfs2
                FsckOpt = "-y"
                )

        Mount db_trsprd_data2 (
                MountPoint = "/db_trsprd/data2"
                BlockDevice = "/dev/trsprd_data2"
                FSType = jfs2
                FsckOpt = "-y"
                )

        Mount db_trsprd_data3 (
                MountPoint = "/db_trsprd/data3"
                BlockDevice = "/dev/trsprd_data3"
                FSType = jfs2
                FsckOpt = "-y"
                )

        Mount db_trsprd_dba (
                MountPoint = "/db_trsprd/dba"
                BlockDevice = "/dev/trsprd_dba"
                FSType = jfs2
                FsckOpt = "-y"
                )

        Mount db_trsprd_oracle (
                MountPoint = "/db_trsprd/oracle"
                BlockDevice = "/dev/trsprd_oracle"
                FSType = jfs2
                FsckOpt = "-y"
                )

        Mount db_trsprd_redo1 (
                MountPoint = "/db_trsprd/redo1"
                BlockDevice = "/dev/trsprd_redo1"
                FSType = jfs2
                FsckOpt = "-y"
                )

        Mount db_trsprd_redo2 (
                MountPoint = "/db_trsprd/redo2"
                BlockDevice = "/dev/trsprd_redo2"
                FSType = jfs2
                FsckOpt = "-y"
                )

        Mount db_trsprd_temp (
                MountPoint = "/db_trsprd/temp"
                BlockDevice = "/dev/trsprd_temp"
                FSType = jfs2
                FsckOpt = "-y"
                )

        Netlsnr trsprd_LSNR (
                Critical = 0
                Owner = oracle
                Home = "/db_trsprd/oracle/product/11.2.0.2"
                TnsAdmin = "/var/opt/oracle"
                Listener = LISTENER_TRSPRD
                EnvFile = "/oracle/home/.profile"
                )

        Oracle trsprd_ORA (
                Critical = 0
                Sid = trsprd
                Owner = oracle
                Home = "/db_trsprd/oracle/product/11.2.0.2"
                EnvFile = "/oracle/home/.profile"
                )

        Proxy trsprd_NIC (
                TargetResName = csgnic
                )

        db_trsprd_arch requires trsprd_ora_lvmvg
        db_trsprd_backup requires trsprd_ora_lvmvg
        db_trsprd_data1 requires trsprd_ora_lvmvg
        db_trsprd_data2 requires trsprd_ora_lvmvg
        db_trsprd_data3 requires trsprd_ora_lvmvg
        db_trsprd_dba requires trsprd_ora_lvmvg
        db_trsprd_oracle requires trsprd_ora_lvmvg
        db_trsprd_redo1 requires trsprd_ora_lvmvg
        db_trsprd_redo2 requires trsprd_ora_lvmvg
        db_trsprd_temp requires trsprd_ora_lvmvg
        trsprd_LSNR requires trsprd_ORA
        trsprd_LSNR requires trsprd_ora_vip
        trsprd_ORA requires db_trsprd_arch
        trsprd_ORA requires db_trsprd_backup
        trsprd_ORA requires db_trsprd_data1
        trsprd_ORA requires db_trsprd_data2
        trsprd_ORA requires db_trsprd_data3
        trsprd_ORA requires db_trsprd_dba
        trsprd_ORA requires db_trsprd_oracle
        trsprd_ORA requires db_trsprd_redo1
        trsprd_ORA requires db_trsprd_redo2
        trsprd_ORA requires db_trsprd_temp
        trsprd_ora_vip requires trsprd_NIC


        // resource dependency tree
        //
        //      group DB_TRS_PRD
        //      {
        //      Netlsnr trsprd_LSNR
        //          {
        //          IP trsprd_ora_vip
        //              {
        //              Proxy trsprd_NIC
        //              }
        //          Oracle trsprd_ORA
        //              {
        //              Mount db_trsprd_temp
        //                  {
        //                  LVMVG trsprd_ora_lvmvg
        //                  }
        //              Mount db_trsprd_backup
        //                  {
        //                  LVMVG trsprd_ora_lvmvg
        //                  }
        //              Mount db_trsprd_arch
        //                  {
        //                  LVMVG trsprd_ora_lvmvg
        //                  }
        //              Mount db_trsprd_redo2
        //                  {
        //                  LVMVG trsprd_ora_lvmvg
        //                  }
        //              Mount db_trsprd_redo1
        //                  {
        //                  LVMVG trsprd_ora_lvmvg
        //                  }
        //              Mount db_trsprd_oracle
        //                  {
        //                  LVMVG trsprd_ora_lvmvg
        //                  }
        //              Mount db_trsprd_dba
        //                  {
        //                  LVMVG trsprd_ora_lvmvg
        //                  }
        //              Mount db_trsprd_data2
        //                  {
        //                  LVMVG trsprd_ora_lvmvg
        //                  }
        //              Mount db_trsprd_data1
        //                  {
        //                  LVMVG trsprd_ora_lvmvg
        //                  }
        //              Mount db_trsprd_data3
        //                  {
        //                  LVMVG trsprd_ora_lvmvg
        //                  }
        //              }
        //          }
        //      }

=======================================================

 

 

 

 

 

  • You would need to identify the reason for the fault of Netlsnr resource. Enable debug logging for Netlsnr resource and then try to online the resource.

    # haconfig -makerw

    # hatype -modify Netlsnr 1 2 3 4 5

    Check the /var/VRTSvcs/log/Netlsnr_A.log and /var/VRTSvcs/log/engine_A.log log files.

    Attach the log if possible.

     

    Regards,

    Sudhir

  • Listener logs : 

     

    tail -1000  /var/VRTSvcs/log/Netlsnr_A.log
    #
    # Log Name:     Netlsnr
    # System:       madagascar
    # SysInfo:      Aix:madagascar,6,1,00F6CEA14C00
    # Created:      2014/03/20 16:40:16
    #

    2014/03/20 16:40:16 VCS INFO V-16-20002-240 Netlsnr:Netlsnr:imf_init:Successfully initialized the driver with reaper:(VCSNetlsnrAgent), pid:(14286978)
    2014/03/20 16:40:16 VCS INFO V-16-2-13805 Thread(1) (imf_init) entry point completed with return status (0)
    2014/03/20 16:42:59 VCS WARNING V-16-2-13112 Thread(1) Unknown resource name trsodsp_LSNR
    2014/03/20 16:43:22 VCS NOTICE V-16-20002-210 Netlsnr:trsodsp_LSNR:monitor:Setting cookie for proc = /db_trsodsp/oracle/product/11.2.0.2/bin/tnslsnr LISTENER_TRSODSP -inherit, PID = 6160414
    2014/03/21 10:07:48 VCS ERROR V-16-2-13066 Thread(1030) Agent is calling clean for resource(trsprd_LSNR) because the resource is not up even after online completed.
    2014/03/21 10:07:50 VCS ERROR V-16-2-13068 Thread(1030) Resource(trsprd_LSNR) - clean completed successfully.
    2014/03/21 10:07:50 VCS ERROR V-16-2-13071 Thread(1030) Resource(trsprd_LSNR): reached OnlineRetryLimit(0).
    2014/03/21 10:11:51 VCS ERROR V-16-2-13066 Thread(1030) Agent is calling clean for resource(trsprd_LSNR) because the resource is not up even after online completed.
    2014/03/21 10:11:53 VCS ERROR V-16-2-13068 Thread(1030) Resource(trsprd_LSNR) - clean completed successfully.
    2014/03/21 10:11:53 VCS ERROR V-16-2-13071 Thread(1030) Resource(trsprd_LSNR): reached OnlineRetryLimit(0).
    2014/03/21 10:22:05 VCS ERROR V-16-2-13066 Thread(1030) Agent is calling clean for resource(trsprd_LSNR) because the resource is not up even after online completed.
    2014/03/21 10:22:07 VCS ERROR V-16-2-13068 Thread(1030) Resource(trsprd_LSNR) - clean completed successfully.
    2014/03/21 10:22:07 VCS ERROR V-16-2-13071 Thread(1030) Resource(trsprd_LSNR): reached OnlineRetryLimit(0).
    2014/03/21 10:35:51 VCS NOTICE V-16-20002-210 Netlsnr:trsprd_LSNR:monitor:Setting cookie for proc = /db_trsprd/oracle/product/11.2.0.2/bin/tnslsnr LISTENER_TRSPRD -inherit, PID = 26279980
    2014/03/21 10:37:27 VCS NOTICE V-16-20002-210 Netlsnr:trsodsp_LSNR:monitor:Setting cookie for proc = /db_trsodsp/oracle/product/11.2.0.2/bin/tnslsnr LISTENER_TRSODSP -inherit, PID = 23134304
    2014/04/05 09:43:33 VCS NOTICE V-16-20002-210 Netlsnr:trsodsp_LSNR:monitor:Setting cookie for proc = /db_trsodsp/oracle/product/11.2.0.2/bin/tnslsnr LISTENER_TRSODSP -inherit, PID = 22872238
    2014/04/05 12:15:05 VCS WARNING V-16-20002-205 Netlsnr:trsodsp_LSNR:monitor:Process tnslsnr restarted
    2014/04/05 11:36:49 VCS INFO V-16-20002-240 Netlsnr:Netlsnr:imf_init:Successfully initialized the driver with reaper:(VCSNetlsnrAgent), pid:(4980806)
    2014/04/05 11:36:49 VCS INFO V-16-2-13805 Thread(1) (imf_init) entry point completed with return status (0)
    2014/04/05 11:54:43 VCS NOTICE V-16-20002-210 Netlsnr:trsodsp_LSNR:monitor:Setting cookie for proc = /db_trsodsp/oracle/product/11.2.0.2/bin/tnslsnr LISTENER_TRSODSP -inherit, PID = 3866828
    2014/04/26 21:17:23 VCS INFO V-16-20002-240 Netlsnr:Netlsnr:imf_init:Successfully initialized the driver with reaper:(VCSNetlsnrAgent), pid:(7012566)
    2014/04/26 21:17:23 VCS INFO V-16-2-13805 Thread(1) (imf_init) entry point completed with return status (0)
    2014/04/26 21:35:51 VCS NOTICE V-16-20002-210 Netlsnr:trsodsp_LSNR:monitor:Setting cookie for proc = /db_trsodsp/oracle/product/11.2.0.2/bin/tnslsnr LISTENER_TRSODSP -inherit, PID = 8912986
    2014/07/19 20:51:26 VCS INFO V-16-20002-240 Netlsnr:Netlsnr:imf_init:Successfully initialized the driver with reaper:(VCSNetlsnrAgent), pid:(9633830)
    2014/07/19 20:51:26 VCS INFO V-16-2-13805 Thread(1) (imf_init) entry point completed with return status (0)
    2014/07/19 21:02:32 VCS NOTICE V-16-20002-210 Netlsnr:trsodsp_LSNR:monitor:Setting cookie for proc = /db_trsodsp/oracle/product/11.2.0.2/bin/tnslsnr LISTENER_TRSODSP -inherit, PID = 6947064
    Thread(772) Error (2) received while fetching psinfo for pid (15728862)2014/09/02 18:51:38 VCS NOTICE V-16-20002-210 Netlsnr:trsprd_LSNR:monitor:Setting cookie for proc = /db_trsprd/oracle/product/11.2.0.2/bin/tnslsnr LISTENER_TRSPRD -inherit, PID = 8192106
    2014/09/02 19:16:15 VCS WARNING V-16-20002-205 Netlsnr:trsodsp_LSNR:monitor:Process tnslsnr restarted
    2015/01/26 11:07:30 VCS NOTICE V-16-20002-210 Netlsnr:trsprd_LSNR:monitor:Setting cookie for proc = /db_trsprd/oracle/product/11.2.0.2/bin/tnslsnr LISTENER_TRSPRD -inherit, PID = 24838262
    2015/02/21 19:08:55 VCS NOTICE V-16-20002-210 Netlsnr:trsodsp_LSNR:monitor:Setting cookie for proc = /db_trsodsp/oracle/product/11.2.0.2/bin/tnslsnr LISTENER_TRSODSP -inherit, PID = 32047200
    2015/02/21 21:14:56 VCS WARNING V-16-20002-205 Netlsnr:trsodsp_LSNR:monitor:Process tnslsnr restarted
    2015/04/11 19:42:54 VCS WARNING V-16-20002-205 Netlsnr:trsprd_LSNR:monitor:Process tnslsnr restarted
    2015/04/11 21:00:28 VCS NOTICE V-16-20002-210 Netlsnr:trsprd_LSNR:monitor:Setting cookie for proc = /db_trsprd/oracle/product/11.2.0.2/bin/tnslsnr LISTENER_TRSPRD -inherit, PID = 17694836
    2015/06/21 23:46:08 VCS NOTICE V-16-20002-210 Netlsnr:trsprd_LSNR:monitor:Setting cookie for proc = /db_trsprd/oracle/product/11.2.0.2/bin/tnslsnr LISTENER_TRSPRD -inherit, PID = 63373516
    2015/07/17 06:17:27 VCS WARNING V-16-2-13108 Thread(1) Engine reported error(e02)
    2015/07/17 06:17:27 VCS WARNING V-16-2-13109 Thread(1) Engine reported error; error info is (Resource does not exist: trsodsp_LSNR)
    2015/08/30 22:54:39 VCS ERROR V-16-2-13066 Thread(772) Agent is calling clean for resource(trsprd_LSNR) because the resource is not up even after online completed.
    2015/08/30 22:54:40 VCS ERROR V-16-2-13068 Thread(772) Resource(trsprd_LSNR) - clean completed successfully.
    2015/08/30 22:54:40 VCS ERROR V-16-2-13071 Thread(772) Resource(trsprd_LSNR): reached OnlineRetryLimit(0).
    2015/08/30 23:02:09 VCS ERROR V-16-2-13066 Thread(772) Agent is calling clean for resource(trsprd_LSNR) because the resource is not up even after online completed.
    2015/08/30 23:02:10 VCS ERROR V-16-2-13068 Thread(772) Resource(trsprd_LSNR) - clean completed successfully.
    2015/08/30 23:02:10 VCS ERROR V-16-2-13071 Thread(772) Resource(trsprd_LSNR): reached OnlineRetryLimit(0).
    2015/08/30 23:05:48 VCS ERROR V-16-2-13066 Thread(772) Agent is calling clean for resource(trsprd_LSNR) because the resource is not up even after online completed.
    2015/08/30 23:05:49 VCS ERROR V-16-2-13068 Thread(772) Resource(trsprd_LSNR) - clean completed successfully.
    2015/08/30 23:05:50 VCS ERROR V-16-2-13071 Thread(772) Resource(trsprd_LSNR): reached OnlineRetryLimit(0).
    2015/08/31 05:53:40 VCS ERROR V-16-2-13066 Thread(772) Agent is calling clean for resource(trsprd_LSNR) because the resource is not up even after online completed.
    2015/08/31 05:53:41 VCS ERROR V-16-2-13068 Thread(772) Resource(trsprd_LSNR) - clean completed successfully.
    2015/08/31 05:53:41 VCS ERROR V-16-2-13071 Thread(772) Resource(trsprd_LSNR): reached OnlineRetryLimit(0).

     

     

    ==============================================================================

     

    engineA.logs :

     

    2015/08/31 05:51:30 VCS INFO V-16-1-10307 Resource trsprd_LSNR (Owner: Unspecified, Group: DB_TRS_PRD) is offline on madagascar (Not initiated by VCS)
    2015/08/31 05:51:39 VCS NOTICE V-16-1-10233 Clearing Restart attribute for group DB_TRS_PRD on all nodes
    2015/08/31 05:51:39 VCS NOTICE V-16-1-10301 Initiating Online of Resource trsprd_LSNR (Owner: Unspecified, Group: DB_TRS_PRD) on System madagascar
    2015/08/31 05:51:39 VCS NOTICE V-16-20002-68 (madagascar) Netlsnr:trsprd_LSNR:online:lsnrctl returned the following output: +----------------------------------
    ----------------------------------+
    2015/08/31 05:51:40 VCS INFO V-16-2-13716 (madagascar) Resource(trsprd_LSNR): Output of the completed operation (online)
    2015/08/31 05:53:40 VCS ERROR V-16-2-13066 (madagascar) Agent is calling clean for resource(trsprd_LSNR) because the resource is not up even after online compl
    eted.
    2015/08/31 05:53:41 VCS CRITICAL V-16-20002-41 (madagascar) Netlsnr:trsprd_LSNR:clean:Listener process LISTENER_TRSPRD not running
    2015/08/31 05:53:41 VCS INFO V-16-2-13068 (madagascar) Resource(trsprd_LSNR) - clean completed successfully.
    2015/08/31 05:53:41 VCS INFO V-16-2-13071 (madagascar) Resource(trsprd_LSNR): reached OnlineRetryLimit(0).
    2015/08/31 05:53:41 VCS ERROR V-16-1-54031 Resource trsprd_LSNR (Owner: Unspecified, Group: DB_TRS_PRD) is FAULTED on sys madagascar
    2015/08/31 05:53:42 VCS INFO V-16-6-15015 (madagascar) hatrigger:/opt/VRTSvcs/bin/triggers/resfault is not a trigger scripts directory or can not be executed

     

  • The engine log suggests that Listener was not able to run successfully.

    2015/08/31 05:51:39 VCS NOTICE V-16-20002-68 (madagascar) Netlsnr:trsprd_LSNR:online:lsnrctl returned the following output: +----------------------------------
    ----------------------------------+

    Usually when the Netlsnr online runs successfully, you will see output similar to below.

    2015/08/07 13:11:38 VCS NOTICE V-16-20002-68 Netlsnr:listsener_res:online:lsnrctl returned the following output: +--------------------------------------------------------------------+
    LD_LIBRARY_PATH - /usr/lib:

    LSNRCTL for Linux: Version 11.2.0.4.0 - Production on 07-AUG-2015 13:11:38

    Copyright (c) 1991, 2013, Oracle.  All rights reserved.

    Starting /oracle/orahome/bin/tnslsnr: please wait...

    TNSLSNR for Linux: Version 11.2.0.4.0 - Production
    System parameter file is /oracle/orahome/network/admin/listener.ora
    Log messages written to /oracle/diag/tnslsnr/vcslx325/listener/alert/log.xml
    Listening on: (DESCRIPTION=(ADDRESS=(PROTOCOL=tcp)(HOST=XXX.xyz.com)(PORT=1521)))
    Listening on: (DESCRIPTION=(ADDRESS=(PROTOCOL=ipc)(KEY=EXTPROC1521)))

    Connecting to (DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=XXX.xyz.com)(PORT=1521)))
    STATUS of the LISTENER
    ------------------------
    Alias                     LISTENER
    Version                   TNSLSNR for Linux: Version 11.2.0.4.0 - Production
    Start Date                07-AUG-2015 13:11:38
    Uptime                    0 days 0 hr. 0 min. 0 sec
    Trace Level               off
    Security                  ON: Local OS Authentication
    SNMP                      OFF
    Listener Parameter File   /oracle/orahome/network/admin/listener.ora
    Listener Log File         /oracle/diag/tnslsnr/XXX/listener/alert/log.xml
    Listening Endpoints Summary...
      (DESCRIPTION=(ADDRESS=(PROTOCOL=tcp)(HOST=XXX.xyz.com)(PORT=1521)))
      (DESCRIPTION=(ADDRESS=(PROTOCOL=ipc)(KEY=EXTPROC1521)))
    The listener supports no services
    The command completed successfully

    Can you enable the debug level and try performing the online operation?

    Also are you able to bring the listener up manually outside VCS control and does VCS report the proper state?

    # su - oracle

    # source /oracle/home/.profile

    # lsnrctl start LISTENER_TRSPRD

    # lsnrctl status LISTENER_TRSPRD

     

    Regards