Sunday, September 28, 2014

Cluster startup error

Service gpnpd failed to come up

Resolution as follows

[root@rac1 bin]# ./crsctl start cluster
CRS-2672: Attempting to start 'ora.evmd' on 'rac1'
CRS-2676: Start of 'ora.evmd' on 'rac1' succeeded
CRS-2672: Attempting to start 'ora.gpnpd' on 'rac1'
CRS-2674: Start of 'ora.gpnpd' on 'rac1' failed
CRS-2673: Attempting to stop 'ora.evmd' on 'rac1'
CRS-2677: Stop of 'ora.evmd' on 'rac1' succeeded
CRS-4000: Command Start failed, or completed with errors.

[root@rac1 bin]# pwd
/u01/app/12.1.0/grid/bin
[root@rac1 bin]# ./crsctl stat res -t -init
--------------------------------------------------------------------------------
Name           Target  State        Server                   State details
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.asm
      1        ONLINE  OFFLINE                               Instance Shutdown,ST
                                                             ABLE
ora.cluster_interconnect.haip
      1        ONLINE  OFFLINE                               STABLE
ora.crf
      1        ONLINE  ONLINE       rac1                     STABLE
ora.crsd
      1        ONLINE  OFFLINE                               STABLE
ora.cssd
      1        ONLINE  OFFLINE                               STABLE
ora.cssdmonitor
      1        ONLINE  OFFLINE                               STABLE
ora.ctssd
      1        ONLINE  OFFLINE                               STABLE
ora.diskmon
      1        ONLINE  OFFLINE                               STABLE
ora.evmd
      1        ONLINE  OFFLINE                               STABLE
ora.gipcd
      1        ONLINE  ONLINE       rac1                     STABLE
ora.gpnpd
      1        ONLINE  OFFLINE                               STABLE
ora.mdnsd
      1        ONLINE  ONLINE       rac1                     STABLE
ora.storage
      1        ONLINE  OFFLINE                               STABLE
--------------------------------------------------------------------------------
[root@rac1 bin]# ./u01/app/11.2.0/grid/bin/ocrcheck
-bash: ./u01/app/11.2.0/grid/bin/ocrcheck: No such file or directory
[root@rac1 bin]# cd /u01/app/11.2.0/grid/bin/
-bash: cd: /u01/app/11.2.0/grid/bin/: No such file or directory
[root@rac1 bin]# pwd
/u01/app/12.1.0/grid/bin
[root@rac1 bin]# ./ocrcheck
Status of Oracle Cluster Registry is as follows :
         Version                  :          4
         Total space (kbytes)     :     409568
         Used space (kbytes)      :       3380
         Available space (kbytes) :     406188
         ID                       : 1384592632
         Device/File Name         :      +DATA
                                    Device/File integrity check succeeded

                                    Device/File not configured

                                    Device/File not configured

                                    Device/File not configured

                                    Device/File not configured

         Cluster registry integrity check succeeded

         Logical corruption check succeeded

[root@rac1 bin]# pwd
/u01/app/12.1.0/grid/bin
You have new mail in /var/spool/mail/root
[root@rac1 bin]# ./crsctl start crs -excl
CRS-2672: Attempting to start 'ora.evmd' on 'rac1'
CRS-2676: Start of 'ora.evmd' on 'rac1' succeeded
CRS-2672: Attempting to start 'ora.gpnpd' on 'rac1'
CRS-2676: Start of 'ora.gpnpd' on 'rac1' succeeded
CRS-2672: Attempting to start 'ora.cssdmonitor' on 'rac1'
CRS-2676: Start of 'ora.cssdmonitor' on 'rac1' succeeded
CRS-2672: Attempting to start 'ora.cssd' on 'rac1'
CRS-2672: Attempting to start 'ora.diskmon' on 'rac1'
CRS-2676: Start of 'ora.diskmon' on 'rac1' succeeded
CRS-2676: Start of 'ora.cssd' on 'rac1' succeeded
CRS-2672: Attempting to start 'ora.cluster_interconnect.haip' on 'rac1'
CRS-2672: Attempting to start 'ora.ctssd' on 'rac1'
CRS-2676: Start of 'ora.ctssd' on 'rac1' succeeded
CRS-5017: The resource action "ora.cluster_interconnect.haip start" encountered the following error:
Start action for HAIP aborted. For details refer to "(:CLSN00107:)" in "/u01/app/grid/diag/crs/rac1/crs/trace/ohasd_orarootagent_root.trc".
CRS-2674: Start of 'ora.cluster_interconnect.haip' on 'rac1' failed
CRS-2679: Attempting to clean 'ora.cluster_interconnect.haip' on 'rac1'
CRS-2681: Clean of 'ora.cluster_interconnect.haip' on 'rac1' succeeded
CRS-2672: Attempting to start 'ora.cluster_interconnect.haip' on 'rac1'
CRS-5017: The resource action "ora.cluster_interconnect.haip start" encountered the following error:
Start action for HAIP aborted. For details refer to "(:CLSN00107:)" in "/u01/app/grid/diag/crs/rac1/crs/trace/ohasd_orarootagent_root.trc".
CRS-2674: Start of 'ora.cluster_interconnect.haip' on 'rac1' failed
CRS-2679: Attempting to clean 'ora.cluster_interconnect.haip' on 'rac1'
CRS-2681: Clean of 'ora.cluster_interconnect.haip' on 'rac1' succeeded
CRS-4000: Command Start failed, or completed with errors.
[root@rac1 bin]# ./crsctl stop resource ora.crsd -init
[root@rac1 bin]# ./crsctl stop resource ora.crsd -init;
CRS-2500: Cannot stop resource 'ora.crsd' as it is not running
CRS-4000: Command Stop failed, or completed with errors.
[root@rac1 bin]# ./crsctl start crs -excl -nocrs
CRS-2672: Attempting to start 'ora.cluster_interconnect.haip' on 'rac1'
CRS-5017: The resource action "ora.cluster_interconnect.haip start" encountered the following error:
Start action for HAIP aborted. For details refer to "(:CLSN00107:)" in "/u01/app/grid/diag/crs/rac1/crs/trace/ohasd_orarootagent_root.trc".
CRS-2674: Start of 'ora.cluster_interconnect.haip' on 'rac1' failed
CRS-2679: Attempting to clean 'ora.cluster_interconnect.haip' on 'rac1'
CRS-2681: Clean of 'ora.cluster_interconnect.haip' on 'rac1' succeeded
CRS-4000: Command Start failed, or completed with errors.
[root@rac1 bin]# pwd
/u01/app/12.1.0/grid/bin
You have new mail in /var/spool/mail/root
[root@rac1 bin]# ./crsctl stat res -t -init
--------------------------------------------------------------------------------
Name           Target  State        Server                   State details
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.asm
      1        ONLINE  OFFLINE                               Instance Shutdown,ST
                                                             ABLE
ora.cluster_interconnect.haip
      1        ONLINE  OFFLINE                               STABLE
ora.crf
      1        ONLINE  ONLINE       rac1                     STABLE
ora.crsd
      1        OFFLINE OFFLINE                               STABLE
ora.cssd
      1        ONLINE  ONLINE       rac1                     STABLE
ora.cssdmonitor
      1        ONLINE  ONLINE       rac1                     STABLE
ora.ctssd
      1        ONLINE  ONLINE       rac1                     OBSERVER,STABLE
ora.diskmon
      1        OFFLINE OFFLINE                               STABLE
ora.evmd
      1        ONLINE  INTERMEDIATE rac1                     STABLE
ora.gipcd
      1        ONLINE  ONLINE       rac1                     STABLE
ora.gpnpd
      1        ONLINE  ONLINE       rac1                     STABLE
ora.mdnsd
      1        ONLINE  ONLINE       rac1                     STABLE
ora.storage
      1        ONLINE  OFFLINE                               STABLE
--------------------------------------------------------------------------------
[root@rac1 bin]# gpnptool get
-bash: gpnptool: command not found

Check the gpnpd profile

[root@rac1 bin]# ./gpnptool get
Warning: some command line parameters were defaulted. Resulting command line:
         ./gpnptool.bin get -o-

<?xml version="1.0" encoding="UTF-8"?><gpnp:GPnP-Profile Version="1.0" xmlns="http://www.grid-pnp.org/2005/11/gpnp-profile" xmlns:gpnp="http://www.grid-pnp.org/2005/11/gpnp-profile" xmlns:orcl="http://www.oracle.com/gpnp/2005/11/gpnp-profile" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.grid-pnp.org/2005/11/gpnp-profile gpnp-profile.xsd" ProfileSequence="6" ClusterUId="7e7ad9a8f8245f05ffad6b38a93508ac" ClusterName="rac-scan" PALocation=""><gpnp:Network-Profile><gpnp:HostNetwork id="gen" HostName="*"><gpnp:Network id="net1" IP="192.168.10.0" Adapter="eth0" Use="cluster_interconnect"/><gpnp:Network id="net2" IP="192.168.56.0" Adapter="eth1" Use="public"/></gpnp:HostNetwork></gpnp:Network-Profile><orcl:CSS-Profile id="css" DiscoveryString="+asm" LeaseDuration="400"/><orcl:ASM-Profile id="asm" DiscoveryString="" SPFile="+DATA/rac-scan/ASMPARAMETERFILE/registry.253.858426557" Mode="legacy"/><ds:Signature xmlns:ds="http://www.w3.org/2000/09/xmldsig#"><ds:SignedInfo><ds:CanonicalizationMethod Algorithm="http://www.w3.org/2001/10/xml-exc-c14n#"/><ds:SignatureMethod Algorithm="http://www.w3.org/2000/09/xmldsig#rsa-sha1"/><ds:Reference URI=""><ds:Transforms><ds:Transform Algorithm="http://www.w3.org/2000/09/xmldsig#enveloped-signature"/><ds:Transform Algorithm="http://www.w3.org/2001/10/xml-exc-c14n#"> <InclusiveNamespaces xmlns="http://www.w3.org/2001/10/xml-exc-c14n#" PrefixList="gpnp orcl xsi"/></ds:Transform></ds:Transforms><ds:DigestMethod Algorithm="http://www.w3.org/2000/09/xmldsig#sha1"/><ds:DigestValue>qD+D2x79xr16FoMfaWXKc1AcYBE=</ds:DigestValue></ds:Reference></ds:SignedInfo><ds:SignatureValue>Yb2cGmHj/flpEvMi4ZzkqUfr/EsPX3yXtqXTraPLG3pffP5IEfwpzF+Q5WQCoYFq5byEmcslYZKl+eNe4Wxog0c183dk5NbAKzdT4S818XD8eh8+XrJP3XtXt2qA2O8FvcLK++1m27Eus7TW+MG/DTUU58V1z2CFyHh9rCF8QaY=</ds:SignatureValue></ds:Signature></gpnp:GPnP-Profile>
Success.

Change the ip address of eth1 and eth0

[root@rac1 bin]# ifconfig eth1 inet 192.168.56.2
[root@rac1 bin]# ifconfig eth0
[root@rac1 bin]# cat /etc/hosts
127.0.0.1                localhost.localdomain localhost
192.168.56.2  rac1.localdomain       rac1
192.168.10.3  rac1-priv.localdomain  rac1-priv
192.168.56.4  rac1-vip.localdomain   rac1-vip
192.168.56.5   rac-scan.localdomain   rac-scan
192.168.56.6  rac2.localdomain       rac2
192.168.10.7  rac2-priv.localdomain  rac2-priv
192.168.56.8  rac2-vip.localdomain   rac2-vip
[root@rac1 bin]# ifconfig eth0 inet 192.168.10.3
[root@rac1 bin]#  ./crsctl start res ora.cluster_interconnect.haip -init
CRS-2672: Attempting to start 'ora.cluster_interconnect.haip' on 'rac1'
CRS-2676: Start of 'ora.cluster_interconnect.haip' on 'rac1' succeeded
CRS-2672: Attempting to start 'ora.asm' on 'rac1'
CRS-2676: Start of 'ora.asm' on 'rac1' succeeded
CRS-2672: Attempting to start 'ora.storage' on 'rac1'
CRS-2676: Start of 'ora.storage' on 'rac1' succeeded
[root@rac1 bin]# ./gpnptool get
Warning: some command line parameters were defaulted. Resulting command line:
         ./gpnptool.bin get -o-

<?xml version="1.0" encoding="UTF-8"?><gpnp:GPnP-Profile Version="1.0" xmlns="http://www.grid-pnp.org/2005/11/gpnp-profile" xmlns:gpnp="http://www.grid-pnp.org/2005/11/gpnp-profile" xmlns:orcl="http://www.oracle.com/gpnp/2005/11/gpnp-profile" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.grid-pnp.org/2005/11/gpnp-profile gpnp-profile.xsd" ProfileSequence="6" ClusterUId="7e7ad9a8f8245f05ffad6b38a93508ac" ClusterName="rac-scan" PALocation=""><gpnp:Network-Profile><gpnp:HostNetwork id="gen" HostName="*"><gpnp:Network id="net1" IP="192.168.10.0" Adapter="eth0" Use="cluster_interconnect"/><gpnp:Network id="net2" IP="192.168.56.0" Adapter="eth1" Use="public"/></gpnp:HostNetwork></gpnp:Network-Profile><orcl:CSS-Profile id="css" DiscoveryString="+asm" LeaseDuration="400"/><orcl:ASM-Profile id="asm" DiscoveryString="" SPFile="+DATA/rac-scan/ASMPARAMETERFILE/registry.253.858426557" Mode="legacy"/><ds:Signature xmlns:ds="http://www.w3.org/2000/09/xmldsig#"><ds:SignedInfo><ds:CanonicalizationMethod Algorithm="http://www.w3.org/2001/10/xml-exc-c14n#"/><ds:SignatureMethod Algorithm="http://www.w3.org/2000/09/xmldsig#rsa-sha1"/><ds:Reference URI=""><ds:Transforms><ds:Transform Algorithm="http://www.w3.org/2000/09/xmldsig#enveloped-signature"/><ds:Transform Algorithm="http://www.w3.org/2001/10/xml-exc-c14n#"> <InclusiveNamespaces xmlns="http://www.w3.org/2001/10/xml-exc-c14n#" PrefixList="gpnp orcl xsi"/></ds:Transform></ds:Transforms><ds:DigestMethod Algorithm="http://www.w3.org/2000/09/xmldsig#sha1"/><ds:DigestValue>qD+D2x79xr16FoMfaWXKc1AcYBE=</ds:DigestValue></ds:Reference></ds:SignedInfo><ds:SignatureValue>Yb2cGmHj/flpEvMi4ZzkqUfr/EsPX3yXtqXTraPLG3pffP5IEfwpzF+Q5WQCoYFq5byEmcslYZKl+eNe4Wxog0c183dk5NbAKzdT4S818XD8eh8+XrJP3XtXt2qA2O8FvcLK++1m27Eus7TW+MG/DTUU58V1z2CFyHh9rCF8QaY=</ds:SignatureValue></ds:Signature></gpnp:GPnP-Profile>
Success.
[root@rac1 bin]# ifconfig -a
eth0      Link encap:Ethernet  HWaddr 08:00:27:BE:F9:01
          inet addr:192.168.10.3  Bcast:192.168.10.255  Mask:255.255.255.0
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:41660 errors:0 dropped:0 overruns:0 frame:0
          TX packets:18776 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:3665536 (3.4 MiB)  TX bytes:4289493 (4.0 MiB)

eth0:1    Link encap:Ethernet  HWaddr 08:00:27:BE:F9:01
          inet addr:169.254.229.41  Bcast:169.254.255.255  Mask:255.255.0.0
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1

eth1      Link encap:Ethernet  HWaddr 08:00:27:3A:51:18
          inet addr:192.168.56.2  Bcast:192.168.56.255  Mask:255.255.255.0
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:24668 errors:0 dropped:0 overruns:0 frame:0
          TX packets:253 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:1825506 (1.7 MiB)  TX bytes:28575 (27.9 KiB)

eth2      Link encap:Ethernet  HWaddr 08:00:27:11:31:D5
          inet addr:10.0.0.19  Bcast:10.0.0.255  Mask:255.255.255.0
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:10052 errors:0 dropped:0 overruns:0 frame:0
          TX packets:123 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:1424473 (1.3 MiB)  TX bytes:20276 (19.8 KiB)

lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          UP LOOPBACK RUNNING  MTU:16436  Metric:1
          RX packets:3126 errors:0 dropped:0 overruns:0 frame:0
          TX packets:3126 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:3800660 (3.6 MiB)  TX bytes:3800660 (3.6 MiB)

[root@rac1 bin]# ifconfig eth1
eth1      Link encap:Ethernet  HWaddr 08:00:27:3A:51:18
          inet addr:192.168.56.2  Bcast:192.168.56.255  Mask:255.255.255.0
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:24709 errors:0 dropped:0 overruns:0 frame:0
          TX packets:259 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:1828108 (1.7 MiB)  TX bytes:31067 (30.3 KiB)

Start the services


[root@rac1 bin]# ./crsctl start res ora.crsd -init
CRS-2672: Attempting to start 'ora.crsd' on 'rac1'
CRS-2676: Start of 'ora.crsd' on 'rac1' succeeded
[root@rac1 bin]# ./crsctl start res ora.cssd -init
CRS-5702: Resource 'ora.cssd' is already running on 'rac1'
CRS-4000: Command Start failed, or completed with errors.
[root@rac1 bin]#


No comments:

Post a Comment

  Diagnosing a long parsing issue in Oracle Database Slide 1: Topic: Diagnosing a Long Parsing Issue in Oracle Database Slide 2: Parsing is ...