无论是物理实体机，还是vmware或virtualbox虚拟机，自己已经安装过N多套10g、11g的rac集群系统， 昨天算是第一次遭遇到PRVF-9992 : Group of device “/dev/mapper/datadg″ did not match the expected group 由于以前在湘潭安装过oracle rac 11.2.0.4 for rhel6.6，安装过程非常顺利，用的方法与这次衡阳的安装也是一模一样，自己感觉到非常纳闷。     操作系统：ORACLE LINUX RHEL6.8     数据库版本：ORACLE 11.2.0.4     物理机器： Dell R730 Server     出现的问题：PRVF 9992 本次安装，集群共享磁盘使用的是multipath映射，修改前的配置： [root@rac1 soft]#cat /etc/multipath.conf # multipath.conf written by anaconda defaults {         user_friendly_names yes } blacklist {         devnode "^(ram|raw|loop|fd|md|dm-|sr|scd|st)[0-9]*"         devnode "^hd[a-z]"         devnode "^dcssblk[0-9]*"         device {                 vendor "DGC"                 product "LUNZ"         }         device {                 vendor "IBM"                 product "S/390.*"         }         # don't count normal SATA devices as multipaths         device {                 vendor  "ATA"         }         # don't count 3ware devices as multipaths         device {                 vendor  "3ware"         }         device {                 vendor  "AMCC"         }         # nor highpoint devices         device {                 vendor  "HPT"         }         wwid "361866da084d8d1002032f80e0b9458e7"         wwid "*" } blacklist_exceptions {         wwid "3600a098000abf3880000025d58a036e7"         wwid "3600a098000abf3880000025658a0367d"         wwid "3600a098000abf3880000026158a0371a"         wwid "3600a098000abf3880000025f58a03705"         wwid "3600a098000abf3880000025b58a036c4"         wwid "3600a098000abf3880000026358a03730" blacklist_exceptions {         wwid "3600a098000abf3880000025d58a036e7"         wwid "3600a098000abf3880000025658a0367d"         wwid "3600a098000abf3880000026158a0371a"         wwid "3600a098000abf3880000025f58a03705"         wwid "3600a098000abf3880000025b58a036c4"         wwid "3600a098000abf3880000026358a03730"         wwid "3600a098000abf3880000025958a036a0" } multipaths {         multipath {                 uid 0                 gid 0                 wwid "3600a098000abf3880000025d58a036e7"                 mode 0600         }         multipath {                 uid 0                 gid 0                 wwid "3600a098000abf3880000025658a0367d"                 mode 0600         }         multipath {                 uid 0                 gid 0                 wwid "3600a098000abf3880000026158a0371a"                 mode 0600         }         multipath {                 uid 0                 gid 0                 wwid "3600a098000abf3880000025f58a03705"                 mode 0600         }         multipath {                 uid 0                 gid 0                 wwid "3600a098000abf3880000025b58a036c4"                 mode 0600         }         multipath {                 uid 0                 gid 0                 wwid "3600a098000abf3880000026358a03730"                 mode 0600         }         multipath {                 uid 0                 gid 0                 wwid "3600a098000abf3880000025958a036a0"                 mode 0600         } } 修改后的配置： # multipath.conf written by anaconda defaults {         user_friendly_names yes } blacklist {         devnode "^(ram|raw|loop|fd|md|dm-|sr|scd|st)[0-9]*"         devnode "^hd[a-z]"         devnode "^dcssblk[0-9]*"         device {                 vendor "DGC"                 product "LUNZ"         }         device {                 vendor "IBM"                 product "S/390.*"         }         # don't count normal SATA devices as multipaths         device {                 vendor  "ATA"         }         # don't count 3ware devices as multipaths         device {                 vendor  "3ware"         }         device {                 vendor  "AMCC"         }         # nor highpoint devices         device {                 vendor  "HPT"         }         wwid "361866da084d8d1002032f80e0b9458e7"         wwid "*" } blacklist_exceptions {         wwid "3600a098000abf3880000025d58a036e7"         wwid "3600a098000abf3880000025658a0367d"         wwid "3600a098000abf3880000026158a0371a"         wwid "3600a098000abf3880000025f58a03705"         wwid "3600a098000abf3880000025b58a036c4"         wwid "3600a098000abf3880000026358a03730" blacklist_exceptions {         wwid "3600a098000abf3880000025d58a036e7"         wwid "3600a098000abf3880000025658a0367d"         wwid "3600a098000abf3880000026158a0371a"         wwid "3600a098000abf3880000025f58a03705"         wwid "3600a098000abf3880000025b58a036c4"         wwid "3600a098000abf3880000026358a03730"         wwid "3600a098000abf3880000025958a036a0" } multipaths { multipath { uid 0 gid 0 wwid "3600a098000a1157d000002d25798865a" mode 0600 } multipath { uid 600 gid 600 alias ocr1 wwid "3600a098000a1157d000002d457988679" mode 0600 } multipath { uid 600 gid 600 alias arch wwid "3600a098000a1157d000002cf57988639" mode 0600 } multipath { uid 0 gid 0 wwid "3600a098000a1157d000002de579886d4" mode 0600 } multipath { uid 600 gid 600 alias data3 wwid "3600a098000a1157d000002e0579886e9" mode 0600 } multipath { uid 600 gid 600 alias ocr2 wwid "3600a098000a1157d000002d65798868a" mode 0600 } multipath { uid 600 gid 600 alias ocr3 wwid "3600a098000a1157d000002d85798869a" mode 0600 } multipath { uid 600 gid 600 alias data1 wwid "3600a098000a1157d000002da579886ac" mode 0600 } multipath { uid 600 gid 600 alias data2 wwid "3600a098000a1157d000002dc579886bf" mode 0600 } } 外加udev修改读写权限： [root@rac1 rules.d]# cat 12-mulitpath-privs.rules  ENV{DM_NAME}=="arch*", OWNER:="grid", GROUP:="asmadmin", MODE:="660" ENV{DM_NAME}=="ocr*", OWNER:="grid", GROUP:="asmadmin", MODE:="660" ENV{DM_NAME}=="data*", OWNER:="grid", GROUP:="asmadmin", MODE:="660" [root@rac1 rules.d]#  主机经过重启，multipath重启，udev重启，再次尝试安装grid，还是有prvf-9992警告；想不到的是：忽略prvf-9992，grid竟然安装成功： [root@rac1 Packages]# /u01/oracle/app/grid/oraInventory/orainstRoot.sh Changing permissions of /u01/oracle/app/grid/oraInventory. Adding read,write permissions for group. Removing read,write,execute permissions for world. Changing groupname of /u01/oracle/app/grid/oraInventory to oinstall. The execution of the script is complete. [root@rac1 Packages]# /u01/oracle/app/grid/home/root.sh Performing root user operation for Oracle 11g  The following environment variables are set as:     ORACLE_OWNER= grid     ORACLE_HOME=  /u01/oracle/app/grid/home Enter the full pathname of the local bin directory: [/usr/local/bin]:     Copying dbhome to /usr/local/bin ...    Copying oraenv to /usr/local/bin ...    Copying coraenv to /usr/local/bin ... Creating /etc/oratab file... Entries will be added to the /etc/oratab file as needed by Database Configuration Assistant when a database is created Finished running generic part of root script. Now product-specific root actions will be performed. Using configuration parameter file: /u01/oracle/app/grid/home/crs/install/crsconfig_params Creating trace directory User ignored Prerequisites during installation Installing Trace File Analyzer OLR initialization - successful   root wallet   root wallet cert   root cert export   peer wallet   profile reader wallet   pa wallet   peer wallet keys   pa wallet keys   peer cert request   pa cert request   peer cert   pa cert   peer root cert TP   profile reader root cert TP   pa root cert TP   peer pa cert TP   pa peer cert TP   profile reader pa cert TP   profile reader peer cert TP   peer user cert   pa user cert Adding Clusterware entries to upstart CRS-2672: Attempting to start 'ora.mdnsd' on 'rac1' CRS-2676: Start of 'ora.mdnsd' on 'rac1' succeeded CRS-2672: Attempting to start 'ora.gpnpd' on 'rac1' CRS-2676: Start of 'ora.gpnpd' on 'rac1' succeeded CRS-2672: Attempting to start 'ora.cssdmonitor' on 'rac1' CRS-2672: Attempting to start 'ora.gipcd' on 'rac1' CRS-2676: Start of 'ora.cssdmonitor' on 'rac1' succeeded CRS-2676: Start of 'ora.gipcd' on 'rac1' succeeded CRS-2672: Attempting to start 'ora.cssd' on 'rac1' CRS-2672: Attempting to start 'ora.diskmon' on 'rac1' CRS-2676: Start of 'ora.diskmon' on 'rac1' succeeded CRS-2676: Start of 'ora.cssd' on 'rac1' succeeded ASM created and started successfully. Disk Group clsdg created successfully. clscfg: -install mode specified Successfully accumulated necessary OCR keys. Creating OCR keys for user 'root', privgrp 'root'.. Operation successful. CRS-4256: Updating the profile Successful addition of voting disk 5edddd8fca634f07bfbf345f7c10f0c2. Successful addition of voting disk 7c64e7be3a1d4f9fbf9f5983896a0adb. Successful addition of voting disk 47924c4f2dda4f36bf3fca4111247e3f. Successfully replaced voting disk group with +clsdg. CRS-4256: Updating the profile CRS-4266: Voting file(s) successfully replaced ##  STATE    File Universal Id                File Name Disk group --  -----    -----------------                --------- ---------  1. ONLINE   5edddd8fca634f07bfbf345f7c10f0c2 (/dev/mapper/crs1) [CLSDG]  2. ONLINE   7c64e7be3a1d4f9fbf9f5983896a0adb (/dev/mapper/crs2) [CLSDG]  3. ONLINE   47924c4f2dda4f36bf3fca4111247e3f (/dev/mapper/crs3) [CLSDG] Located 3 voting disk(s). CRS-2672: Attempting to start 'ora.asm' on 'rac1' CRS-2676: Start of 'ora.asm' on 'rac1' succeeded CRS-2672: Attempting to start 'ora.CLSDG.dg' on 'rac1' CRS-2676: Start of 'ora.CLSDG.dg' on 'rac1' succeeded Configure Oracle Grid Infrastructure for a Cluster ... succeeded [root@rac1 Packages]# [root@rac2 ~]# /u01/oracle/app/grid/oraInventory/orainstRoot.sh Changing permissions of /u01/oracle/app/grid/oraInventory. Adding read,write permissions for group. Removing read,write,execute permissions for world. Changing groupname of /u01/oracle/app/grid/oraInventory to oinstall. The execution of the script is complete. [root@rac2 ~]# /u01/oracle/app/grid/home/root.sh Performing root user operation for Oracle 11g  The following environment variables are set as:     ORACLE_OWNER= grid     ORACLE_HOME=  /u01/oracle/app/grid/home Enter the full pathname of the local bin directory: [/usr/local/bin]:     Copying dbhome to /usr/local/bin ...    Copying oraenv to /usr/local/bin ...    Copying coraenv to /usr/local/bin ... Creating /etc/oratab file... Entries will be added to the /etc/oratab file as needed by Database Configuration Assistant when a database is created Finished running generic part of root script. Now product-specific root actions will be performed. Using configuration parameter file: /u01/oracle/app/grid/home/crs/install/crsconfig_params Creating trace directory User ignored Prerequisites during installation Installing Trace File Analyzer OLR initialization - successful Adding Clusterware entries to upstart CRS-4402: The CSS daemon was started in exclusive mode but found an active CSS daemon on node rac1, number 1, and is terminating An active cluster was found during exclusive startup, restarting to join the cluster Configure Oracle Grid Infrastructure for a Cluster ... succeeded [root@rac2 ~]#  [grid@rac1 ~]$ crs_stat -t Name           Type           Target    State     Host         ------------------------------------------------------------ ora.CLSDG.dg   ora....up.type ONLINE    ONLINE    rac1         ora....N1.lsnr ora....er.type ONLINE    ONLINE    rac1         ora.asm        ora.asm.type   ONLINE    ONLINE    rac1         ora.cvu        ora.cvu.type   ONLINE    ONLINE    rac1         ora.gsd        ora.gsd.type   OFFLINE   OFFLINE                ora....network ora....rk.type ONLINE    ONLINE    rac1         ora.oc4j       ora.oc4j.type  ONLINE    ONLINE    rac1         ora.ons        ora.ons.type   ONLINE    ONLINE    rac1         ora....SM1.asm application    ONLINE    ONLINE    rac1         ora.rac1.gsd   application    OFFLINE   OFFLINE                ora.rac1.ons   application    ONLINE    ONLINE    rac1         ora.rac1.vip   ora....t1.type ONLINE    ONLINE    rac1         ora....SM2.asm application    ONLINE    ONLINE    rac2         ora.rac2.gsd   application    OFFLINE   OFFLINE                ora.rac2.ons   application    ONLINE    ONLINE    rac2         ora.rac2.vip   ora....t1.type ONLINE    ONLINE    rac2         ora.scan1.vip  ora....ip.type ONLINE    ONLINE    rac1         [grid@rac1 ~]$  [grid@rac1 ~]$ crsctl stat res -t -------------------------------------------------------------------------------- NAME           TARGET  STATE        SERVER                   STATE_DETAILS        -------------------------------------------------------------------------------- Local Resources -------------------------------------------------------------------------------- ora.CLSDG.dg                ONLINE  ONLINE       rac1                                                         ONLINE  ONLINE       rac2                                          ora.asm                ONLINE  ONLINE       rac1                     Started                             ONLINE  ONLINE       rac2                     Started              ora.gsd                OFFLINE OFFLINE      rac1                                                         OFFLINE OFFLINE      rac2                                          ora.net1.network                ONLINE  ONLINE       rac1                                                         ONLINE  ONLINE       rac2                                          ora.ons                ONLINE  ONLINE       rac1                                                         ONLINE  ONLINE       rac2                                          -------------------------------------------------------------------------------- Cluster Resources -------------------------------------------------------------------------------- ora.LISTENER_SCAN1.lsnr       1        ONLINE  ONLINE       rac1                                          ora.cvu       1        ONLINE  ONLINE       rac1                                          ora.oc4j       1        ONLINE  ONLINE       rac1                                          ora.rac1.vip       1        ONLINE  ONLINE       rac1                                          ora.rac2.vip       1        ONLINE  ONLINE       rac2                                          ora.scan1.vip       1        ONLINE  ONLINE       rac1                                          [grid@rac1 ~]$  [grid@rac1 ~]$ lsnrctl status racscan LSNRCTL for Linux: Version 11.2.0.4.0 - Production on 13-FEB-2017 22:26:25 Copyright (c) 1991, 2013, Oracle.  All rights reserved. Connecting to (DESCRIPTION=(CONNECT_DATA=(SERVICE_NAME=))(ADDRESS=(PROTOCOL=TCP)(HOST=10.141.101.144)(PORT=1521))) STATUS of the LISTENER ------------------------ Alias                     LISTENER_SCAN1 Version                   TNSLSNR for Linux: Version 11.2.0.4.0 - Production Start Date                13-FEB-2017 22:18:10 Uptime                    0 days 0 hr. 8 min. 15 sec Trace Level               off Security                  ON: Local OS Authentication SNMP                      OFF Listener Parameter File   /u01/oracle/app/grid/home/network/admin/listener.ora Listener Log File         /u01/oracle/app/grid/home/log/diag/tnslsnr/rac1/listener_scan1/alert/log.xml Listening Endpoints Summary...   (DESCRIPTION=(ADDRESS=(PROTOCOL=ipc)(KEY=LISTENER_SCAN1)))   (DESCRIPTION=(ADDRESS=(PROTOCOL=tcp)(HOST=10.141.101.144)(PORT=1521))) The listener supports no services The command completed successfully [grid@rac1 ~]$      另一个问题是：DBCA建库遭遇ORA-19504&ORA-15001 开始给DATADG一共4T的共享存储，但是grid通过asmca查看时显示totalsize是4TB，而freesize只有2TB； 然后，将datadg拆开成datadg1、datadg2,每个2T，再次通过asmca查看时显示totalsize和freesize一致了。 但是，重试dbca建库的时候还是到9%报错，依然是：ORA-19504&ORA-15001。怀疑是mutipath问题，将映射配置改为： multipath { uid 600 gid 600 alias arch wwid "3600a098000a1157d000002cf57988639" mode 0660 path_grouping_policy    failover } 并且重新手工同步主机时钟（由于操作系统不是自己安装的，集群grid安装成功后，查看操作系统时钟，2台主机时间相差10多分钟）， 然后， grid关闭集群，重启操作系统后，oracle用户再次尝试dbca成功执行： [grid@rac2 ~]$ crs_stat -t Name           Type           Target    State     Host         ------------------------------------------------------------ ora.ARCHDG.dg  ora....up.type ONLINE    ONLINE    rac1         ora.CLSDG.dg   ora....up.type ONLINE    ONLINE    rac1         ora.DATADG1.dg ora....up.type ONLINE    ONLINE    rac1         ora.DATADG2.dg ora....up.type ONLINE    ONLINE    rac1         ora....ER.lsnr ora....er.type ONLINE    ONLINE    rac1         ora....N1.lsnr ora....er.type ONLINE    ONLINE    rac2         ora.asm        ora.asm.type   ONLINE    ONLINE    rac1         ora.cvu        ora.cvu.type   ONLINE    ONLINE    rac1         ora.gsd        ora.gsd.type   OFFLINE   OFFLINE                ora....network ora....rk.type ONLINE    ONLINE    rac1         ora.oc4j       ora.oc4j.type  ONLINE    ONLINE    rac1         ora.ons        ora.ons.type   ONLINE    ONLINE    rac1         ora.orcl.db    ora....se.type ONLINE    ONLINE    rac1         ora....SM1.asm application    ONLINE    ONLINE    rac1         ora....C1.lsnr application    ONLINE    ONLINE    rac1         ora.rac1.gsd   application    OFFLINE   OFFLINE                ora.rac1.ons   application    ONLINE    ONLINE    rac1         ora.rac1.vip   ora....t1.type ONLINE    ONLINE    rac1         ora....SM2.asm application    ONLINE    ONLINE    rac2         ora....C2.lsnr application    ONLINE    ONLINE    rac2         ora.rac2.gsd   application    OFFLINE   OFFLINE                ora.rac2.ons   application    ONLINE    ONLINE    rac2         ora.rac2.vip   ora....t1.type ONLINE    ONLINE    rac2         ora.scan1.vip  ora....ip.type ONLINE    ONLINE    rac2         [grid@rac2 ~]$  [oracle@rac1 ~]$ sqlplus / as sysdba SQL*Plus: Release 11.2.0.4.0 Production on Mon Feb 13 18:34:39 2017 Copyright (c) 1982, 2013, Oracle.  All rights reserved. Connected to: Oracle Database 11g Enterprise Edition Release 11.2.0.4.0 - 64bit Production With the Partitioning, Real Application Clusters, Automatic Storage Management, OLAP, Data Mining and Real Application Testing options SQL> select open_mode from v$database; OPEN_MODE -------------------- READ WRITE SQL>  SQL> select status from v$instance; STATUS ------------ OPEN SQL>  [grid@rac1 ~]$ crsctl stat res -t -------------------------------------------------------------------------------- NAME           TARGET  STATE        SERVER                   STATE_DETAILS        -------------------------------------------------------------------------------- Local Resources -------------------------------------------------------------------------------- ora.ARCHDG.dg                ONLINE  ONLINE       rac1                                                         ONLINE  ONLINE       rac2                                          ora.CLSDG.dg                ONLINE  ONLINE       rac1                                                         ONLINE  ONLINE       rac2                                          ora.DATADG1.dg                ONLINE  ONLINE       rac1                                                         ONLINE  ONLINE       rac2                                          ora.DATADG2.dg                ONLINE  ONLINE       rac1                                                         ONLINE  ONLINE       rac2                                          ora.LISTENER.lsnr                ONLINE  ONLINE       rac1                                                         ONLINE  ONLINE       rac2                                          ora.asm                ONLINE  ONLINE       rac1                     Started                             ONLINE  ONLINE       rac2                     Started              ora.gsd                OFFLINE OFFLINE      rac1                                                         OFFLINE OFFLINE      rac2                                          ora.net1.network                ONLINE  ONLINE       rac1                                                         ONLINE  ONLINE       rac2                                          ora.ons                ONLINE  ONLINE       rac1                                                         ONLINE  ONLINE       rac2                                          -------------------------------------------------------------------------------- Cluster Resources -------------------------------------------------------------------------------- ora.LISTENER_SCAN1.lsnr       1        ONLINE  ONLINE       rac2                                          ora.cvu       1        ONLINE  ONLINE       rac1                                          ora.oc4j       1        ONLINE  ONLINE       rac1                                          ora.orcl.db       1        ONLINE  ONLINE       rac1                     Open                       2        ONLINE  ONLINE       rac2                     Open                 ora.rac1.vip       1        ONLINE  ONLINE       rac1                                          ora.rac2.vip       1        ONLINE  ONLINE       rac2                                          ora.scan1.vip       1        ONLINE  ONLINE       rac2                                          [grid@rac1 ~]$