1.数据库启动集群报错
[root@db1 bin]# ./crs_stat -t -vName Type R/RA F/FT Target State Host ----------------------------------------------------------------------ora.BACKUP.dg ora....up.type 0/5 0/ ONLINE ONLINE db1 ora.DATA.dg ora....up.type 0/5 0/ ONLINE ONLINE db1 ora....ER.lsnr ora....er.type 0/5 0/ ONLINE OFFLINE ora....N1.lsnr ora....er.type 0/5 0/0 ONLINE OFFLINE ora.OCR.dg ora....up.type 0/5 0/ ONLINE ONLINE db1 ora.asm ora.asm.type 0/5 0/ ONLINE ONLINE db1 ora.cvu ora.cvu.type 0/5 0/0 ONLINE ONLINE db1 ora....SM1.asm application 0/5 0/0 ONLINE ONLINE db1 ora....B1.lsnr application 0/5 0/0 ONLINE OFFLINE ora.db1.gsd application 0/5 0/0 OFFLINE OFFLINE ora.db1.ons application 0/3 0/0 ONLINE OFFLINE ora.db1.vip ora....t1.type 0/0 0/0 ONLINE ONLINE db1 ora....SM2.asm application 0/5 0/0 ONLINE ONLINE db2 ora....B2.lsnr application 0/5 0/0 ONLINE OFFLINE ora.db2.gsd application 0/5 0/0 OFFLINE OFFLINE ora.db2.ons application 0/3 0/0 ONLINE OFFLINE ora.db2.vip ora....t1.type 0/0 0/0 ONLINE ONLINE db2 ora.gsd ora.gsd.type 0/5 0/ OFFLINE OFFLINE ora....network ora....rk.type 0/5 0/ ONLINE ONLINE db1 ora.oc4j ora.oc4j.type 0/1 0/2 ONLINE OFFLINE ora.ons ora.ons.type 0/3 0/ ONLINE OFFLINE ora.racdb.db ora....se.type 0/2 0/1 ONLINE OFFLINE ora....ry.acfs ora....fs.type 0/5 0/ ONLINE ONLINE db1 ora.scan1.vip ora....ip.type 0/0 0/0 ONLINE OFFLINE [root@db1 bin]# ./srvctl status listenerListener LISTENER is enabledListener LISTENER is not running[root@db1 bin]# ./srvctl start listenerPRCR-1079 : Failed to start resource ora.LISTENER.lsnrCRS-5016: Process "/u01/app/11.2.0/grid/bin/lsnrctl" spawned by agent "/u01/app/11.2.0/grid/bin/oraagent.bin" for action "start" failed: details at "(:CLSN00010:)" in "/u01/app/11.2.0/grid/log/db2/agent/crsd/oraagent_grid/oraagent_grid.log"CRS-5016: Process "/u01/app/11.2.0/grid/bin/lsnrctl" spawned by agent "/u01/app/11.2.0/grid/bin/oraagent.bin" for action "start" failed: details at "(:CLSN00010:)" in "/u01/app/11.2.0/grid/log/db1/agent/crsd/oraagent_grid/oraagent_grid.log"CRS-5016: Process "/u01/app/11.2.0/grid/bin/lsnrctl" spawned by agent "/u01/app/11.2.0/grid/bin/oraagent.bin" for action "start" failed: details at "(:CLSN00010:)" in "/u01/app/11.2.0/grid/log/db2/agent/crsd/oraagent_grid/oraagent_grid.log"CRS-5016: Process "/u01/app/11.2.0/grid/bin/lsnrctl" spawned by agent "/u01/app/11.2.0/grid/bin/oraagent.bin" for action "start" failed: details at "(:CLSN00010:)" in "/u01/app/11.2.0/grid/log/db1/agent/crsd/oraagent_grid/oraagent_grid.log"CRS-2674: Start of 'ora.LISTENER.lsnr' on 'db2' failedCRS-2674: Start of 'ora.LISTENER.lsnr' on 'db1' failed
[root@db1 oraagent_grid]# tail -50 oraagent_grid.log
2018-05-22 09:53:56.551: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] Adding Environment Variables TNS_ADMIN=/u01/app/11.2.0/grid/network/admin/2018-05-22 09:53:56.551: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] Adding Environment variable from USR_ORA_ENV ORACLE_BASE=/u01/app/grid2018-05-22 09:53:56.551: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] Utils:execCmd action = 3 flags = 38 ohome = (null) cmdname = lsnrctl. 2018-05-22 09:53:56.551: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] getOracleHomeAttrib: oracle_home = /u01/app/11.2.0/grid2018-05-22 09:53:56.653: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)2018-05-22 09:53:56.653: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)LSNRCTL for Linux: Version 11.2.0.4.0 - Production on 22-MAY-2018 09:53:562018-05-22 09:53:56.653: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)2018-05-22 09:53:56.653: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)Copyright (c) 1991, 2013, Oracle. All rights reserved.2018-05-22 09:53:56.653: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)2018-05-22 09:53:56.653: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)Connecting to (DESCRIPTION=(ADDRESS=(PROTOCOL=IPC)(KEY=LISTENER)))2018-05-22 09:53:56.653: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)2018-05-22 09:53:58.255: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)TNS-12541: TNS:no listener2018-05-22 09:53:58.255: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:) TNS-12560: TNS:protocol adapter error2018-05-22 09:53:58.255: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:) TNS-00511: No listener2018-05-22 09:53:58.255: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:) Linux Error: 111: Connection refused2018-05-22 09:53:58.255: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)2018-05-22 09:53:58.256: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)Utils:execCmd scls_process_join() uret 12018-05-22 09:53:58.256: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] execCmd ret = 1
2018-05-22 09:53:58.256: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)2018-05-22 09:53:58.256: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)LSNRCTL for Linux: Version 11.2.0.4.0 - Production on 22-MAY-2018 09:53:562018-05-22 09:53:58.256: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)2018-05-22 09:53:58.256: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)Copyright (c) 1991, 2013, Oracle. All rights reserved.2018-05-22 09:53:58.256: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)2018-05-22 09:53:58.256: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)Connecting to (DESCRIPTION=(ADDRESS=(PROTOCOL=IPC)(KEY=LISTENER)))2018-05-22 09:53:58.256: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)TNS-12541: TNS:no listener2018-05-22 09:53:58.256: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:) TNS-12560: TNS:protocol adapter error2018-05-22 09:53:58.256: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:) TNS-00511: No listener2018-05-22 09:53:58.256: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:) Linux Error: 111: Connection refused2018-05-22 09:53:58.256: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)2018-05-22 09:53:58.256: [ AGFW][3501381376]{1:4489:169} ora.LISTENER.lsnr db1 1 state changed from: STARTING to: OFFLINE2018-05-22 09:53:58.257: [ AGFW][3501381376]{1:4489:169} Agent sending last reply for: RESOURCE_START[ora.LISTENER.lsnr db1 1] ID 4098:8502018-05-22 09:54:07.297: [CLSFRAME][3522594592] TM [MultiThread] is changing desired thread # to 5. Current # is 62018-05-22 09:54:07.298: [CLSFRAME][2876454656]{0:1:8} Worker thread is exiting in TM [MultiThread] to meet the desired count of 5. New count is 52018-05-22 09:54:37.120: [ AGFW][3501381376]{1:4489:2} Agent received the message: AGENT_HB[Engine] ID 12293:9562018-05-22 09:54:37.120: [ AGFW][3501381376]{1:4489:2} Deleting the resource: ora.LISTENER_SCAN1.lsnr 1 12018-05-22 09:54:37.272: [ora.asm][2882758400]{1:4489:2} [check] AsmProxyAgent:setAttrsFromSrcRes 2 m_oracleSid +ASM1 cmdId 260 checkType 3 flag 322018-05-22 09:54:37.272: [ora.asm][2882758400]{1:4489:2} [check] CrsCmd::ClscrsCmdData::stat entity 1 statflag 32 useFilter 02018-05-22 09:54:37.299: [ USRTHRD][2882758400]{1:4489:2} Thread:ASM DedicatedThreadstart { 2018-05-22 09:54:37.299: [ USRTHRD][2882758400]{1:4489:2} Thread:ASM DedicatedThreadstart }2018-05-22 09:54:37.299: [ USRTHRD][2876454656]{1:4489:2} ASM Dedicated Thread { 2018-05-22 09:54:37.299: [ USRTHRD][2876454656]{1:4489:2} AsmProxyAgent::setGenUsrOraInst updating GEN_USR_ORA_INST_NAME@SERVERNAME(db1) to +ASM12018-05-22 09:54:37.299: [ora.asm][2882758400]{1:4489:2} [check] CrsCmd::ClscrsCmdData::stat entity 1 statflag 33 useFilter 02018-05-22 09:54:37.300: [ USRTHRD][2876454656]{1:4489:2} CrsCmd::ClscrsCmdData::stat entity 1 statflag 32 useFilter 02018-05-22 09:54:37.320: [ora.asm][2882758400]{1:4489:2} [check] AsmProxyAgent::check clsagfw_res_status 02018-05-22 09:54:37.327: [ USRTHRD][2876454656]{1:4489:2} setResAttrib: attr GEN_USR_ORA_INST_NAME@SERVERNAME(db1) clsagfw attribVal +ASM1 clscrs tmpAttrValue +ASM12018-05-22 09:54:37.327: [ USRTHRD][2876454656]{1:4489:2} setResAttrib clsagfw_modify_attribute attr GEN_USR_ORA_INST_NAME@SERVERNAME(db1) value +ASM1 retCode 02018-05-22 09:54:37.327: [ USRTHRD][2876454656]{1:4489:2} ASM Dedicated Thread }2018-05-22 09:54:37.327: [ USRTHRD][2876454656]{1:4489:2} Thread:ASM DedicatedThreadisRunning is reset to false here2018-05-22 09:55:07.122: [ AGFW][3501381376]{1:4489:2} Agent received the message: AGENT_HB[Engine] ID 12293:975查看/etc/hosts文件，发现127.0.0.1被注释，打开注释，重启crs
[root@db1 oraagent_grid]# cat /etc/hosts#127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4#::1 localhost localhost.localdomain localhost6 localhost6.localdomain6[root@db1 oraagent_grid]# vi /etc/hosts#127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4#::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
[root@db1 oraagent_grid]# cd /u01/app/11.2.0/grid/bin[root@db1 bin]# ./crsctl stop cluster -allCRS-2673: Attempting to stop 'ora.crsd' on 'db1'CRS-2673: Attempting to stop 'ora.crsd' on 'db2'CRS-2790: Starting shutdown of Cluster Ready Services-managed resources on 'db1'CRS-2673: Attempting to stop 'ora.scan1.vip' on 'db1'CRS-2673: Attempting to stop 'ora.db1.vip' on 'db1'CRS-2673: Attempting to stop 'ora.OCR.dg' on 'db1'CRS-2673: Attempting to stop 'ora.registry.acfs' on 'db1'CRS-2673: Attempting to stop 'ora.racdb.db' on 'db1'CRS-2673: Attempting to stop 'ora.oc4j' on 'db1'CRS-2673: Attempting to stop 'ora.cvu' on 'db1'CRS-2790: Starting shutdown of Cluster Ready Services-managed resources on 'db2'CRS-2673: Attempting to stop 'ora.db2.vip' on 'db2'CRS-2673: Attempting to stop 'ora.OCR.dg' on 'db2'CRS-2673: Attempting to stop 'ora.registry.acfs' on 'db2'CRS-2673: Attempting to stop 'ora.racdb.db' on 'db2'CRS-2677: Stop of 'ora.cvu' on 'db1' succeededCRS-2677: Stop of 'ora.db2.vip' on 'db2' succeededCRS-2677: Stop of 'ora.racdb.db' on 'db1' succeededCRS-2673: Attempting to stop 'ora.BACKUP.dg' on 'db1'CRS-2673: Attempting to stop 'ora.DATA.dg' on 'db1'CRS-2677: Stop of 'ora.racdb.db' on 'db2' succeededCRS-2673: Attempting to stop 'ora.BACKUP.dg' on 'db2'CRS-2673: Attempting to stop 'ora.DATA.dg' on 'db2'CRS-2677: Stop of 'ora.registry.acfs' on 'db2' succeededCRS-2677: Stop of 'ora.registry.acfs' on 'db1' succeededCRS-2677: Stop of 'ora.scan1.vip' on 'db1' succeededCRS-2677: Stop of 'ora.BACKUP.dg' on 'db1' succeededCRS-2677: Stop of 'ora.DATA.dg' on 'db2' succeededCRS-2677: Stop of 'ora.DATA.dg' on 'db1' succeededCRS-2677: Stop of 'ora.BACKUP.dg' on 'db2' succeededCRS-2677: Stop of 'ora.db1.vip' on 'db1' succeededCRS-2677: Stop of 'ora.oc4j' on 'db1' succeededCRS-2677: Stop of 'ora.OCR.dg' on 'db2' succeededCRS-2673: Attempting to stop 'ora.asm' on 'db2'CRS-2677: Stop of 'ora.OCR.dg' on 'db1' succeededCRS-2673: Attempting to stop 'ora.asm' on 'db1'CRS-2677: Stop of 'ora.asm' on 'db2' succeededCRS-2673: Attempting to stop 'ora.net1.network' on 'db2'CRS-2677: Stop of 'ora.net1.network' on 'db2' succeededCRS-2792: Shutdown of Cluster Ready Services-managed resources on 'db2' has completedCRS-2677: Stop of 'ora.asm' on 'db1' succeededCRS-2673: Attempting to stop 'ora.net1.network' on 'db1'CRS-2677: Stop of 'ora.net1.network' on 'db1' succeededCRS-2792: Shutdown of Cluster Ready Services-managed resources on 'db1' has completedCRS-2677: Stop of 'ora.crsd' on 'db2' succeededCRS-2673: Attempting to stop 'ora.ctssd' on 'db2'CRS-2673: Attempting to stop 'ora.evmd' on 'db2'CRS-2673: Attempting to stop 'ora.asm' on 'db2'CRS-2677: Stop of 'ora.crsd' on 'db1' succeededCRS-2673: Attempting to stop 'ora.ctssd' on 'db1'CRS-2673: Attempting to stop 'ora.evmd' on 'db1'CRS-2673: Attempting to stop 'ora.asm' on 'db1'CRS-2677: Stop of 'ora.evmd' on 'db2' succeededCRS-2677: Stop of 'ora.evmd' on 'db1' succeededCRS-2677: Stop of 'ora.ctssd' on 'db2' succeededCRS-2677: Stop of 'ora.ctssd' on 'db1' succeededCRS-2677: Stop of 'ora.asm' on 'db2' succeededCRS-2673: Attempting to stop 'ora.cluster_interconnect.haip' on 'db2'CRS-2677: Stop of 'ora.asm' on 'db1' succeededCRS-2673: Attempting to stop 'ora.cluster_interconnect.haip' on 'db1'CRS-2677: Stop of 'ora.cluster_interconnect.haip' on 'db1' succeededCRS-2673: Attempting to stop 'ora.cssd' on 'db1'CRS-2677: Stop of 'ora.cluster_interconnect.haip' on 'db2' succeededCRS-2673: Attempting to stop 'ora.cssd' on 'db2'CRS-2677: Stop of 'ora.cssd' on 'db1' succeededCRS-2677: Stop of 'ora.cssd' on 'db2' succeeded[root@db1 bin]# ./crsctl start cluster -allCRS-2672: Attempting to start 'ora.cssdmonitor' on 'db1'CRS-2672: Attempting to start 'ora.cssdmonitor' on 'db2'CRS-2676: Start of 'ora.cssdmonitor' on 'db2' succeededCRS-2672: Attempting to start 'ora.cssd' on 'db2'CRS-2672: Attempting to start 'ora.diskmon' on 'db2'CRS-2676: Start of 'ora.diskmon' on 'db2' succeededCRS-2676: Start of 'ora.cssdmonitor' on 'db1' succeededCRS-2672: Attempting to start 'ora.cssd' on 'db1'CRS-2672: Attempting to start 'ora.diskmon' on 'db1'CRS-2676: Start of 'ora.diskmon' on 'db1' succeededCRS-2676: Start of 'ora.cssd' on 'db2' succeededCRS-2672: Attempting to start 'ora.ctssd' on 'db2'CRS-2676: Start of 'ora.cssd' on 'db1' succeededCRS-2672: Attempting to start 'ora.ctssd' on 'db1'CRS-2672: Attempting to start 'ora.cluster_interconnect.haip' on 'db1'CRS-2676: Start of 'ora.ctssd' on 'db2' succeededCRS-2672: Attempting to start 'ora.evmd' on 'db2'CRS-2672: Attempting to start 'ora.cluster_interconnect.haip' on 'db2'CRS-2676: Start of 'ora.ctssd' on 'db1' succeededCRS-2672: Attempting to start 'ora.evmd' on 'db1'CRS-2676: Start of 'ora.evmd' on 'db2' succeededCRS-2676: Start of 'ora.evmd' on 'db1' succeededCRS-2676: Start of 'ora.cluster_interconnect.haip' on 'db1' succeededCRS-2672: Attempting to start 'ora.asm' on 'db1'CRS-2676: Start of 'ora.cluster_interconnect.haip' on 'db2' succeededCRS-2672: Attempting to start 'ora.asm' on 'db2'CRS-2676: Start of 'ora.asm' on 'db1' succeededCRS-2672: Attempting to start 'ora.crsd' on 'db1'CRS-2676: Start of 'ora.asm' on 'db2' succeededCRS-2672: Attempting to start 'ora.crsd' on 'db2'CRS-2676: Start of 'ora.crsd' on 'db1' succeededCRS-2676: Start of 'ora.crsd' on 'db2' succeeded[root@db1 bin]# ./crs_stat -t -vName Type R/RA F/FT Target State Host ----------------------------------------------------------------------ora.BACKUP.dg ora....up.type 0/5 0/ ONLINE ONLINE db1 ora.DATA.dg ora....up.type 0/5 0/ ONLINE ONLINE db1 ora....ER.lsnr ora....er.type 0/5 0/ ONLINE ONLINE db1 ora....N1.lsnr ora....er.type 0/5 0/0 ONLINE ONLINE db1 ora.OCR.dg ora....up.type 0/5 0/ ONLINE ONLINE db1 ora.asm ora.asm.type 0/5 0/ ONLINE ONLINE db1 ora.cvu ora.cvu.type 0/5 0/0 ONLINE ONLINE db1 ora....SM1.asm application 0/5 0/0 ONLINE ONLINE db1 ora....B1.lsnr application 0/5 0/0 ONLINE ONLINE db1 ora.db1.gsd application 0/5 0/0 OFFLINE OFFLINE ora.db1.ons application 0/3 0/0 ONLINE ONLINE db1 ora.db1.vip ora....t1.type 0/0 0/0 ONLINE ONLINE db1 ora....SM2.asm application 0/5 0/0 ONLINE ONLINE db2 ora....B2.lsnr application 0/5 0/0 ONLINE ONLINE db2 ora.db2.gsd application 0/5 0/0 OFFLINE OFFLINE ora.db2.ons application 0/3 0/0 ONLINE ONLINE db2 ora.db2.vip ora....t1.type 0/0 0/0 ONLINE ONLINE db2 ora.gsd ora.gsd.type 0/5 0/ OFFLINE OFFLINE ora....network ora....rk.type 0/5 0/ ONLINE ONLINE db1 ora.oc4j ora.oc4j.type 0/1 0/2 ONLINE ONLINE db1 ora.ons ora.ons.type 0/3 0/ ONLINE ONLINE db1 ora.racdb.db ora....se.type 0/2 0/1 ONLINE ONLINE db1 ora....ry.acfs ora....fs.type 0/5 0/ ONLINE ONLINE db1 ora.scan1.vip ora....ip.type 0/0 0/0 ONLINE ONLINE db1