一、环境介绍

服务器

vSphere 6虚拟机

操作系统

OEL 7.0 with Kernal 3.8.13-35.3.1.el7uek

硬件配置

CPU 32 Cores

内存 96 G

硬盘 150 G


二、IP规划

主机名

Public IP

Private IP

VIP

Scan IP

oraclerac8281

10.116.82.81

192.168.82.81

10.116.82.84

10.116.82.87

oraclerac8282

10.116.82.82

192.168.82.82

10.116.82.85

oraclerac8283

10.116.82.83

192.168.82.83

10.116.82.86


三、存储规划

硬盘

ASM Disk类型

大小

/dev/sdb

OCR/Voting Disk/GIMR

50G

/dev/sdc

OCR/Voting Disk/GIMR

50G

/dev/sdd

OCR/Voting Disk/GIMR

50G

/dev/sde

DATA

2T

/dev/sdf

RECOVERY

1T


四、系统配置(每个节点)

1、禁用Selinux/Firewall

sed -i '/^SELINUX=.*/ s//SELINUX=disabled/'/etc/selinux/config

echo "Disable Firewall"

systemctl disable firewalld

2、配置本地Yum

cat > /etc/yum.repos.d/local.repo<

[rhel-media]

name=RHEL-\$releasever - Media

baseurl=file:///media

enabled=1

gpgcheck=0

gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release

!

3、配置Host解析

sed -i 3,\$d /etc/hosts

cat >> /etc/hosts <

#Public IP

10.116.82.81     oraclerac8281

10.116.82.82     oraclerac8282

10.116.82.83     oraclerac8283

#Private IP

192.168.82.81    oraclerac8281-priv

192.168.82.82    oraclerac8282-priv

192.168.82.83    oraclerac8283-priv

#VIP

10.116.82.84     oraclerac8281-vip

10.116.82.85     oraclerac8282-vip

10.116.82.86     oraclerac8283-vip

#Scan-IP

10.116.82.87     oraclerac-act-scan

!

4、依赖包安装

yum -y install binutils compat-libcap1compat-libstdc++-33 \

 compat-libstdc++-33*.i686 gcc gcc-c++ glibcglibc*.i686 \

 glibc-devel glibc-devel*.i686 ksh libaiolibaio-*.i686 \

 libaio-devel libaio-devel*.i686 libgcclibgcc*.i686 \

 libstdc++ libstdc++-*.i686 libstdc++-devellibstdc++devel*.i686 \

 libXi libXi*.i686 libXtst libXtst*.i686 makesysstat \

 unixODBC unixODBC-devel unixODBC*.i686zlib-devel zlib-devel.i686 \

 nfs-utilssysstat chrony smartmontools net-tools zip unzip

5、系统参数配置

MEMTOTAL=$(free -b | sed -n '2p' | awk'{print $2}')

let SHMMAX=$MEMTOTAL*4/5

SHMMNI=4096

let SHMALL=$MEMTOTAL/4096

echo SHMMAX:$SHMMAX SHMALL:$SHMALL

cat > /etc/sysctl.d/98-oracle.conf<

fs.aio-max-nr = 1048576

fs.file-max = 6815744

kernel.shmall = $SHMALL

kernel.shmmax = $SHMMAX

kernel.shmmni = 4096

kernel.sem = 250 32000 100 128

net.ipv4.ip_local_port_range = 9000 65500

net.core.rmem_default = 262144

net.core.rmem_max = 4194304

net.core.wmem_default = 262144

net.core.wmem_max = 1048576

kernel.panic_on_oops = 1

net.ipv4.conf.all.rp_filter = 2

net.ipv4.conf.default.rp_filter = 2

!

/sbin/sysctl -p /etc/sysctl.d/98-oracle.conf

6、添加用户和组

groupadd -g 54321 oinstall

groupadd -g 54322 dba

groupadd -g 54323 oper

groupadd -g 54324 backupdba

groupadd -g 54325 dgdba

groupadd -g 54326 kmdba

groupadd -g 54327 asmdba

groupadd -g 54328 asmoper

groupadd -g 54329 asmadmin

groupadd -g 54330 racdba

useradd -u 54321 -g oinstall -Gdba,asmdba,backupdba,dgdba,kmdba,racdba,oper oracle

useradd -u 54331 -g oinstall -Gdba,oper,backupdba,dgdba,kmdba,asmdba,asmoper,asmadmin,racdba grid

echo "grid" | passwd --stdin grid

echo "oracle:oracle" | chpasswd

7、创建目录并设置权限

mkdir -p /u01/app/grid

mkdir -p /u01/app/12.2.0/grid

mkdir -p /u01/app/oracle

mkdir -p/u01/app/oracle/product/12.2.0/db_1

mkdir -p /u01/app/oraInventory

chown -R grid:oinstall /u01/app

chown -R oracle:oinstall /u01/app/oracle

chmod -R 775 /u01/app

8、修改用户安全限制

cat >/etc/security/limits.d/oracle-database-server-12cR2-preinstall.conf <

oracle    soft     nofile   2048

oracle    hard     nofile   65536

oracle    soft     nproc    16384

oracle    hard     nproc    16384

oracle    soft     stack    10243

oracle    hard     stack    32768

oracle    hard     memlock  134217728

oracle    soft     memlock  134217728

grid      soft     nofile   2048

grid      hard     nofile   65536

grid      soft     nproc    16384

grid      hard     nproc    16384

grid      soft     stack    10243

grid      hard     stack    32768

grid      hard     memlock  134217728

grid      soft     memlock  134217728
!

9、添加PAM模块

cat >> /etc/pam.d/login <

session  required    pam_limits.so

!

10、禁用透明大页

cat >> /etc/rc.local <

if test -f/sys/kernel/mm/redhat_transparent_hugepage/enabled; then

 echo never > /sys/kernel/mm/redhat_transparent_hugepage/enabled

fi

!

11、配置Grid环境变量

sed -i 13,\$d ~grid/.bash_profile

cat >> ~grid/.bash_profile <

export ORACLE_BASE=/u01/app/grid

export ORACLE_HOME=/u01/app/12.2.0/grid

export ORACLE_SID=+ASM$(STR=$(hostname);echo ${STR: -1})

exportPATH=\$ORACLE_HOME/bin:\$ORACLE_HOME/OPatch:\$PATH

exportLD_LIBRARY_PATH=\$ORACLE_HOME/lib:\$LD_LIBRARY_PATH

!

cat ~grid/.bash_profile

12、配置Oracle环境变量

sed -i 13,\$d ~oracle/.bash_profile

cat >> ~oracle/.bash_profile<

export ORACLE_BASE=/u01/app/oracle

exportORACLE_HOME=\$ORACLE_BASE/product/12.2.0/db_1

export ORACLE_SID=orcl1$(STR=$(hostname);echo ${STR: -1})

export NLS_LANG=AMERICAN_AMERICA.AL32UTF8

exportPATH=\$ORACLE_HOME/bin:\$ORACLE_HOME/OPatch:\$PATH

exportLD_LIBRARY_PATH=\$ORACLE_HOME/lib:\$LD_LIBRARY_PATH

!

cat ~oracle/.bash_profile

13、配置Oracle/Grid互信(只需在一个节点执行)

Oracle提供的database安装包里有脚本可以配置互信,比较方便。

su - oracle -c "unzip -qo /tmp/linuxx64_12201_database.zip-d /u01/app"

/u01/app/database/sshsetup/sshUserSetup.sh-user oracle -hosts "oraclerac8281 oraclerac8282 oraclerac8283"-advanced -noPromptPassphrase

/u01/app/database/sshsetup/sshUserSetup.sh-user grid -hosts "oraclerac8281 oraclerac8282 oraclerac8283"-advanced -noPromptPassphrase

配置完成后使用下面命令测试用户连通性

su - oracle -c "ssh oraclerac8281 date;ssh oraclerac8282 date; ssh oraclerac8283 date; exit"

su - grid -c "ssh oraclerac8281 date; sshoraclerac8282 date; ssh oraclerac8283 date; exit"

分发cvuqdisk安装包到所有节点/tmp

su - oracle -c "

scp /u01/app/database/rpm/cvuqdisk-1.0.10-1.rpmoraclerac8281:/tmp

scp /u01/app/database/rpm/cvuqdisk-1.0.10-1.rpmoraclerac8282:/tmp

scp /u01/app/database/rpm/cvuqdisk-1.0.10-1.rpmoraclerac8283:/tmp

"

14、配置NOZEROCONF

echo "NOZEROCONF=yes" >>/etc/sysconfig/network

grep ^NOZEROCONF /etc/sysconfig/network

15、修改nsswitch.conf

sed -i "/^hosts:/chosts:      files dns nis" /etc/nsswitch.conf

grep ^hosts /etc/nsswitch.conf

16、配置tmpfs挂载大小

默认tmpfs挂载的大小为内存的一半,SGA设置如果超过内存一半,需要重新配置。

mem=$(free -m | grep ^Mem | awk'{print$2}'); let sga=mem/100*64

sed -i /^tmpfs/d /etc/fstab

echo "tmpfs /dev/shm tmpfs size=${sga}M,mode=1777,nosuid,nodev,strictatime0 0" >> /etc/fstab

mount -o loop /dev/shm

df -h /dev/shm

17、安装CVUDisk

export CVUQDISK_GRP=asmadmin

rpm -ivh /tmp/cvuqdisk-1.0.10-1.rpm

18、桌面环境安装(只需要装一个节点)

如果RAC服务器和本地节点全端口互通,只需要安装X Window即可,否则需要安装桌面环境

X Window安装"(可选)

yum -y groupinstall "X Windowsystem"

Gnome桌面环境安装(可选)

yum -y groupinstall "Server withGUI"

五、UDEV配置

1、获取DiskUUID

虚拟机需要关机,设置disk.EnableUUID=TRUE

虚拟机属性->选项->高级->常规->配置参数


2、配置UDEVRule文件

使用上一步获取到的UUID一一对应替换下面RESULT值,并自定义挂载名称(标红部分)。

cat >/etc/udev/rules.d/99-oracle-asmdevices.rules <

KERNEL=="sd*",ENV{DEVTYPE}=="disk", SUBSYSTEM=="block",PROGRAM=="/usr/lib/udev/scsi_id -g -u -d \$devnode",RESULT=="36000c29d6d2dd44e3dc77ec91b3c4b27", RUN+="/bin/sh -c'mknod /dev/asmdisk01 b \$major \$minor; chown grid:asmadmin /dev/asmdisk01; chmod 0660 /dev/asmdisk01'"

KERNEL=="sd*",ENV{DEVTYPE}=="disk", SUBSYSTEM=="block",PROGRAM=="/usr/lib/udev/scsi_id -g -u -d \$devnode",RESULT=="36000c29f112c5cc79f547b0571cf17e6", RUN+="/bin/sh -c'mknod /dev/asmdisk02 b \$major \$minor; chown grid:asmadmin /dev/asmdisk02; chmod 0660 /dev/asmdisk02'"

KERNEL=="sd*",ENV{DEVTYPE}=="disk", SUBSYSTEM=="block",PROGRAM=="/usr/lib/udev/scsi_id -g -u -d \$devnode",RESULT=="36000c29e113772582ecb184c62e6f470", RUN+="/bin/sh -c'mknod /dev/asmdisk03 b \$major \$minor; chown grid:asmadmin /dev/asmdisk03; chmod 0660 /dev/asmdisk03'"

KERNEL=="sd*",ENV{DEVTYPE}=="disk", SUBSYSTEM=="block",PROGRAM=="/usr/lib/udev/scsi_id -g -u -d \$devnode",RESULT=="36000c290e51112c33800622a77a3cc0e", RUN+="/bin/sh -c'mknod /dev/datdisk01 b \$major \$minor; chown grid:asmadmin /dev/datdisk01; chmod 0660 /dev/datdisk01'"

KERNEL=="sd*",ENV{DEVTYPE}=="disk", SUBSYSTEM=="block", PROGRAM=="/usr/lib/udev/scsi_id-g -u -d \$devnode",RESULT=="36000c29783cf6791a19b69403dd9a064", RUN+="/bin/sh -c'mknod /dev/arcdisk01 b \$major \$minor; chown grid:asmadmin /dev/arcdisk01; chmod 0660 /dev/arcdisk01'"

!

cat >> /etc/udev/rules.d/99-oracle-asmdevices.rules<

ACTION=="add|change",ENV{ID_SERIAL}=="36000c29d6d2dd44e3dc77ec91b3c4b27", NAME="asmdisk01",  OWNER="grid",GROUP="asmadmin", MODE="0660"

ACTION=="add|change",ENV{ID_SERIAL}=="36000c29f112c5cc79f547b0571cf17e6", NAME="asmdisk02",  OWNER="grid",GROUP="asmadmin", MODE="0660"

ACTION=="add|change",ENV{ID_SERIAL}=="36000c29e113772582ecb184c62e6f470", NAME="asmdisk03",  OWNER="grid",GROUP="asmadmin", MODE="0660"

ACTION=="add|change",ENV{ID_SERIAL}=="36000c290e51112c33800622a77a3cc0e", NAME="datdisk01",  OWNER="grid",GROUP="asmadmin", MODE="0660"

ACTION=="add|change",ENV{ID_SERIAL}=="36000c29783cf6791a19b69403dd9a064", NAME="arcdisk01",  OWNER="grid",GROUP="asmadmin", MODE="0660"

!

3、重新加载规则

udevadm control --reload-rules

/sbin/udevadm trigger --type=devices--action=change

sleep 3

ls -Ll /dev/sd*

ls -Ll /dev/*disk0*

确认Owner/Group正确。

# ls -Ll /dev/sd*

brw-rw---- 1 root disk     8, 0 Apr 24  2020 /dev/sda

brw-rw---- 1 root disk     8, 1 Apr 24  2020 /dev/sda1

brw-rw---- 1 root disk     8,  2Apr 24  2020 /dev/sda2

brw-rw---- 1 grid asmadmin 8, 16 Apr24  2020 /dev/sdb

brw-rw---- 1 grid asmadmin 8, 32 Apr24  2020 /dev/sdc

brw-rw---- 1 grid asmadmin 8, 48 Apr24  2020 /dev/sdd

brw-rw---- 1 grid asmadmin 8, 64 Apr24  2020 /dev/sde

brw-rw---- 1 grid asmadmin 8, 80 Apr24  2020 /dev/sdf

# ls /dev/*disk0* -l

brw-rw---- 1 grid asmadmin 8, 80 Apr 2322:35 /dev/arcdisk01

brw-rw---- 1 grid asmadmin 8, 16 Apr 2322:35 /dev/asmdisk01

brw-rw---- 1 grid asmadmin 8, 32 Apr 2322:35 /dev/asmdisk02

brw-rw---- 1 grid asmadmin 8, 48 Apr 2322:35 /dev/asmdisk03

brw-rw---- 1 grid asmadmin 8, 64 Apr 2322:35 /dev/datdisk01

插曲:一定要确保原始盘的属主配置正确,否则在GI执行root.sh18步的时候会出现CLSRSC-182: Failed to add the resource for ASMdiskgroup的错误。

六、安装GridInfrastructure(节点1执行)

1、解压

因为从12c R2开始,Grid组件以解压安装的方式,所以需要将压缩包直接解压到$ORACLE_HOME目录,否则解压的目录就是安装目录了。

unzip -q /tmp/linuxx64_12201_grid_home.zip-d $ORACLE_HOME

2、检测安装环境

$ORACLE_HOME/runcluvfy.sh  stage -pre crsinst -n oraclerac8281,oraclerac8282,oraclerac8283 -fixup -verbose

3、开始安装

$ORACLE_HOME/gridSetup.sh

4、执行脚本(root

/u01/app/oraInventory/orainstRoot.sh

/u01/app/12.2.0/grid/root.sh

等节点1执行成功之后,在其它节点并行执行。

5、检查集群状态

crsctl status res -t

6、创建其它磁盘组

asmca

七、安装Database(节点1执行)

1、安装

/u01/database/runInstaller

2、执行脚本(root

/u01/app/oracle/product/12.2.0/db_1/root.sh

3、检查数据库集群状态

srvctl status database -d orcl

附录

1ORACLE 12.2认证的OS软件版本

序号

组件

1

Oracle Linux 7 with the Unbreakable Enterprise kernel:
3.8.13-35.3.1.el7uek.x86_64 or later

2

Oracle Linux 7 with the Red Hat Compatible kernel:
3.10.0-123.el7.x86_64 or later

3

Oracle Linux 6.4 with the Unbreakable Enterprise kernel 2:
2.6.39-200.24.1.el6uek.x86_64 or later

4

Oracle Linux 6 with the Red Hat Compatible kernel:
2.6.32-358.el6.x86_64 or later

5

Red Hat Enterprise Linux 7: 3.10.0-123.el7.x86_64 or later

6

Red Hat Enterprise Linux 6: 2.6.32-358.el6.x86_64 or later

7

SUSE Linux Enterprise Server 12: 3.12.28-4-default or later

8

SUSE Linux Enterprise Server 11 SP2: 3.0.13-0.27 or later


2、集群管理命令

启动Cluster集群(root)

/u01/app/12.2.0/grid/bin/crsctl startcluster [-all | -n ]

关闭Cluster集群(root)

/u01/app/12.2.0/grid/bin/crsctl startcluster [-all | -n ]

-all             启动集群内所有节点

-n  启动指定节点

查看集群状态(grid)

crsctl status res -t

查看数据库集群状态(oracle)

srvctl status database -d orcl

查看Listener状态

srvctl status listener

关闭实例

srvctl stop instance -d orcl -n

启动实例

srvctl start instance -d orcl -n

关闭数据库

srvctl stop database -d orcl

启动数据库

srvctl start database -d orcl

01-05 22:55