一、OS规划
1、系统规划

2、网络规划
- 每个节点必须具有至少两个以上得网卡,一张网卡对外提供网络服务,另一张网卡用于各个节点间得通信和心跳检测等(内部通信)
- 需要保证Public-IP、VIP-IP、Scan-IP在同一网段上,其中Private-IP位于另一网段上(该网段只要独立即可)
- 当前配置的网卡在虚拟机中新增两张,一张为NAT,一张为桥接模式。当新建OS后默认会存在一张网卡,所以只需要新增一张即可
- 虚拟机新增网卡需要关机进行
编辑私网网卡
bash
# cd /etc/sysconfig/network-scripts/
# cp ifcfg-ens32 ifcfg-ens34
# vi ifcfg-ens34
TYPE=Ethernet
PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO=static
DEFROUTE=no -- 改为 no
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
NAME=ens34 -- 更名为ip addr默认识别名
#UUID=2f2ea75f-c499-4b28-9fef-15cb3b4f3743 -- 注释掉
DEVICE=ens34 -- 更名为ip addr默认识别名
ONBOOT=yes
NM_CONTROLLED=yes
IPADDR=192.168.1.33 -- 不同网段(VIP)
NETMASK=255.255.255.0
GATEWAY=192.168.1.2 -- 与IP保持同一网段
编辑公网网卡
bash
# vi ifcfg-ens32
TYPE=Ethernet
PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO= static
DEFROUTE=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
NAME=ens32
#UUID=2f2ea75f-c499-4b28-9fef-15cb3b4f3743 -- 注释掉
DEVICE=ens32
ONBOOT=yes
IPADDR=192.168.136.33
NETMASK=255.255.255.0
GATEWAY=192.168.136.2
DNS=223.5.5.5
3、IP及主机名更改(所有节点执行)
bash
# IP设置(两节点)
$ vi /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
# public ip
192.168.136.33 rac1.localdomain.com rac1
192.168.136.63 rac2.localdomain.com rac2
# private-ip
192.168.1.33 rac1-priv
192.168.1.34 rac2-priv
# node-vip
192.168.136.133 rac1-node1-vip
192.168.136.134 rac2-node2-vip
# scan ip
192.168.136.140 rac1-scan
4、关闭防火墙(所有节点执行)
bash
$ systemctl stop firewalld.service && systemctl disable firewalld.service
5、修改selinux文件(所有节点执行)
bash
# SELINUX=disabled ---> vi /etc/selinux/config
6、安装依赖
bash
yum groupinstall -y "Server with GUI"
yum install -y bc \
binutils \
compat-libcap1 \
compat-libstdc++-33 \
gcc \
gcc-c++ \
elfutils-libelf \
elfutils-libelf-devel \
glibc \
glibc-devel \
ksh \
libaio \
libaio-devel \
libgcc \
libstdc++ \
libstdc++-devel \
libxcb \
libX11 \
libXau \
libXi \
libXtst \
libXrender \
libXrender-devel \
make \
net-tools \
nfs-utils \
smartmontools \
sysstat \
e2fsprogs \
e2fsprogs-libs \
fontconfig-devel \
expect \
unzip \
openssh-clients \
readline* \
tigervnc* \
psmisc --skip-broken
7、修改限制文件(所有节点执行)
bash
# vi /etc/security/limits.conf
oracle soft nproc 2047
oracle hard nproc 16384
oracle soft nofile 4096
oracle hard nofile 65536
oracle soft stack 10240
8、新增用户组并追加到用户(所有节点执行)
bash
groupadd oper
groupadd backupdba
groupadd dgdba
groupadd kmdba
groupadd asmdba
groupadd asmoper
groupadd asmadmin
groupadd racdba
usermod -G asmdba,oper oracle
useradd -g oinstall -G dba,oper,backupdba,dgdba,kmdba,asmdba,asmoper,asmadmin,racdba -d /home/grid -m grid
9、共享盘安装
注意:此处将会出一版完全安装版。
10、新增文件夹并更改授权(所有节点执行)
bash
mkdir -p /data/app/11.2.0/grid
mkdir -p /data/app/grid
chown -R grid:oinstall /data/
mkdir -p /data/app/oracle
chown oracle:oinstall /data/app/oracle
chmod -R 775 /data/
11、更改密码(所有节点执行)
bash
echo "oracle" | passwd --stdin oracle
echo "grid" | passwd --stdin grid
12、配置环境变量(所有节点执行)
bash
$ su -- grid
$ vi .bash_profile
# .bash_profile
# User specific environment and startup programs
export LANG=en_US.UTF-8
export CVUQDISK_GRP=asmadmin
export ORACLE_BASE=/data/app/grid
export ORACLE_HOME=/data/app/11.2.0/grid
# node 2 : +ASM2
export ORACLE_SID=+ASM1
export TNS_ADMIN=$ORACLE_HOME/network/admin
# export NLS_LANG="SIMPLIFIED CHINESE_CHINA.ZHS16GBK"
export NLS_LANG="AMERICAN_AMERICA.AL32UTF8";
#export NLS_LANG="SIMPLIFIED CHINESE_CHINA.AL32UTF8";
# export LD_LIBRARY_PATH=$ORACLE_HOME/lib:$ORACLE_HOME/oracm/lib:/lib:/usr/lib:/usr/local/li b
export PATH=${PATH}:$ORACLE_HOME/bin:$ORACLE_HOME/OPatch
export TMPDIR=/tmp
set -o vi
umask 022
$ su - oracle
$ vi .bash_profile
# .bash_profile
# User specific environment and startup programs
export LANG=en_US.UTF-8
export ORACLE_BASE=/data/app/oracle
export ORACLE_HOME=$ORACLE_BASE/product/11.2.0.4/db_1
export ORACLE_SID=rac1
export ORACLE_UNQNAME=rac
export TNS_ADMIN=$ORACLE_HOME/network/admin
# export ORA_NLS33=$ORACLE_HOME/ocommon/nls/admin/data
# export NLS_LANG="SIMPLIFIED CHINESE_CHINA.ZHS16GBK"
export NLS_LANG="AMERICAN_AMERICA.AL32UTF8";
#export LD_LIBRARY_PATH=$ORACLE_HOME/lib:$ORACLE_HOME/network/lib:/lib:/usr/lib #export LIBPATH=$ORACLE_HOME/lib:$ORACLE_HOME/network/lib:/lib:/usr/lib
export NLS_DATE_FORMAT="yyyy-mm-dd hh24:mi:ss"
export PATH=.:${PATH}:$ORACLE_HOME/bin:$ORACLE_HOME/OPatch
export PATH=${PATH}:/usr/bin:/bin:/usr/bin/X11:/usr/local/bin
export TMPDIR=/tmp
set -o vi
export EDITOR=vi
export ALERT_HOME=$ORACLE_BASE/diag/rdbms/$ORACLE_SID/trace
13、配置ssh互信(所有节点执行)
注意:此处仅配置Oracle和grid用户的互信。所有节点均需要执行,包括当前节点当前用户。如:有一个主节点,两个从节点。那么每台服务器每个用户需要执行三次互信
bash
-- grid用户两节点互信
su - grid
ssh-keygen -t rsa
ssh-copy-id -i .ssh/id_rsa.pub grid@[主节点]
ssh-copy-id -i .ssh/id_rsa.pub grid@[从节点]
-- oracle用户两节点互信
su - oracle
ssh-keygen -t rsa
ssh-copy-id -i .ssh/id_rsa.pub oracle@[主节点]
ssh-copy-id -i .ssh/id_rsa.pub oracle@[从节点]
-- 校验脚本
注意:执行该命令时不需要输入密码即可。只要下面的结果一致即可
$ while read line;do ssh -oStrictHostKeyChecking=no $line "echo ok" </dev/null ;done < <(cat /etc/hosts|grep ^1|grep -v 127|grep -v vip|grep -v scan|sed 's/ /\n/g'|sort|uniq|sed '1d')
Warning: Permanently added '' (ECDSA) to the list of known hosts.
ok
ok
ok
ok
Warning: Permanently added 'master' (ECDSA) to the list of known hosts.
ok
Warning: Permanently added 'master-priv' (ECDSA) to the list of known hosts.
ok
ok
Warning: Permanently added 'standby' (ECDSA) to the list of known hosts.
ok
Warning: Permanently added 'standby-priv' (ECDSA) to the list of known hosts.
ok
14、新建中央仓库(所有节点执行)
bash
$ vi /etc/oraInst.loc
inventory_loc=/data/oraInventory
inst_group=oinstall
mkdir -p /data/oraInventory
chown -R grid:oinstall /data/oraInventory
chmod -R 777 /data/oraInventory
二、安装GRID软件
1、下载软件并解压(所有节点执行)
bash
# 软件包
注意:当前软件包可以在SR界面中patche界面输入13390677下载
p13390677_112040_Linux-x86-64_1of7.zip
p13390677_112040_Linux-x86-64_2of7.zip
p13390677_112040_Linux-x86-64_3of7.zip
# 解压
$ for i p*.zip;
>do
>unzip --o $i --d software
>done;
2、测试连通性(所有节点执行)
bash
# 安装测试包
$ cd /data/Download/software/grid/rpm
$ rpm --ivh cvuqdisk-1.0.9-1.rpm
# 测试连通性
$ su - grid
$ cd /data/Download/software/grid
$ ./runcluvfy.sh stage -post hwos -n rac1,rac2 -verbose
# 注意:在所有节点均执行。如果检测出任何错误,请先修复再执行安装。
[grid@rac2 grid]$ ./runcluvfy.sh stage -post hwos -n rac1,rac2 -verbose
Performing post-checks for hardware and operating system setup
Checking node reachability...
Check: Node reachability from node "rac2"
Destination Node Reachable?
------------------------------------ ------------------------
rac1 yes
rac2 yes
Result: Node reachability check passed from node "rac2"
Checking user equivalence...
Check: User equivalence for user "grid"
Node Name Status
------------------------------------ ------------------------
rac2 passed
rac1 passed
Result: User equivalence check passed for user "grid"
Checking node connectivity...
Checking hosts config file...
Node Name Status
------------------------------------ ------------------------
rac2 passed
rac1 passed
Verification of the hosts config file successful
Interface information for node "rac1"
Name IP Address Subnet Gateway Def. Gateway HW Address MTU
------ --------------- --------------- --------------- --------------- ----------------- ------
ens192 192.168.136.33 192.168.136.0 UNKNOWN UNKNOWN 00:0C:29:D4:49:36 1500
ens224 192.168.1.33 192.168.1.0 UNKNOWN UNKNOWN 00:0C:29:D4:49:40 1500
Interface information for node "rac2"
Name IP Address Subnet Gateway Def. Gateway HW Address MTU
------ --------------- --------------- --------------- --------------- ----------------- ------
ens192 192.168.136.63 192.168.136.0 UNKNOWN UNKNOWN 00:0C:29:F3:A4:2B 1500
ens224 192.168.1.34 192.168.1.0 UNKNOWN UNKNOWN 00:0C:29:F3:A4:35 1500
Check: Node connectivity of subnet "192.168.136.0"
Source Destination Connected?
------------------------------ ------------------------------ ----------------
rac2[192.168.136.63] rac1[192.168.136.33] yes
Result: Node connectivity passed for subnet "192.168.136.0" with node(s) rac2,rac1
Check: TCP connectivity of subnet "192.168.136.0"
Source Destination Connected?
------------------------------ ------------------------------ ----------------
rac2:192.168.136.63 rac1:192.168.136.33 passed
Result: TCP connectivity check passed for subnet "192.168.136.0"
Check: Node connectivity of subnet "192.168.1.0"
Source Destination Connected?
------------------------------ ------------------------------ ----------------
rac2[192.168.1.34] rac1[192.168.1.33] yes
Result: Node connectivity passed for subnet "192.168.1.0" with node(s) rac2,rac1
Check: TCP connectivity of subnet "192.168.1.0"
Source Destination Connected?
------------------------------ ------------------------------ ----------------
rac2:192.168.1.34 rac1:192.168.1.33 passed
Result: TCP connectivity check passed for subnet "192.168.1.0"
Interfaces found on subnet "192.168.136.0" that are likely candidates for a private interconnect are:
rac2 ens192:192.168.136.63
rac1 ens192:192.168.136.33
Interfaces found on subnet "192.168.1.0" that are likely candidates for a private interconnect are:
rac2 ens192:192.168.1.34
rac1 ens192:192.168.1.33
WARNING:
Could not find a suitable set of interfaces for VIPs
Checking subnet mask consistency...
Subnet mask consistency check passed for subnet "192.168.136.0".
Subnet mask consistency check passed for subnet "192.168.1.0".
Subnet mask consistency check passed.
Result: Node connectivity check passed
Checking multicast communication...
Checking subnet "192.168.136.0" for multicast communication with multicast group "230.0.1.0"...
Check of subnet "192.168.136.0" for multicast communication with multicast group "230.0.1.0" passed.
Checking subnet "192.168.1.0" for multicast communication with multicast group "230.0.1.0"...
Check of subnet "192.168.1.0" for multicast communication with multicast group "230.0.1.0" passed.
Check of multicast communication passed.
Checking for multiple users with UID value 0
Result: Check for multiple users with UID value 0 passed
Check: Time zone consistency
Result: Time zone consistency check passed
Checking shared storage accessibility...
WARNING:
rac1:Unable to determine the shareability of device /dev/sda on nodes:
rac2,rac1
Disk Sharing Nodes (2 in count)
------------------------------------ ------------------------
/dev/sdb rac2 rac1
Disk Sharing Nodes (2 in count)
------------------------------------ ------------------------
/dev/sdc rac2 rac1
Disk Sharing Nodes (2 in count)
------------------------------------ ------------------------
/dev/sdd rac2 rac1
Disk Sharing Nodes (2 in count)
------------------------------------ ------------------------
/dev/sde rac2 rac1
Disk Sharing Nodes (2 in count)
------------------------------------ ------------------------
/dev/sdf rac2 rac1
Disk Sharing Nodes (2 in count)
------------------------------------ ------------------------
/dev/sdg rac2 rac1
Disk Sharing Nodes (2 in count)
------------------------------------ ------------------------
/dev/sdh rac2 rac1
Shared storage check was successful on nodes "rac2,rac1"
Checking integrity of name service switch configuration file "/etc/nsswitch.conf" ...
Checking if "hosts" entry in file "/etc/nsswitch.conf" is consistent across nodes...
Checking file "/etc/nsswitch.conf" to make sure that only one "hosts" entry is defined
More than one "hosts" entry does not exist in any "/etc/nsswitch.conf" file
All nodes have same "hosts" entry defined in file "/etc/nsswitch.conf"
Check for integrity of name service switch configuration file "/etc/nsswitch.conf" passed
Post-check for hardware and operating system setup was successful.
3、安装软件(任一节点即可)
注意:只需要在某一个节点执行安装,默认会复制到其它所有节点。所以其它节点均不需要独立安装
bash
$ su -- grid
$ cd /data/Download/software/grid/
$ export DISPLAY=[IP]:0.0
$ ./ runInstaller


bash
# 补丁编号:18370031
重点:所有节点均需要执行
# 更改权限并解压
$ chown grid:oinstall p18370031_112040_Linux-x86-64.zip
$ su - grid
$ unzip p18370031_112040_Linux-x86-64.zip
# 应用补丁
$ cd /data/app/11.2.0/grid/OPatch
$ ./opatch napply -oh /data/app/11.2.0/grid/ -local /data/app/patch/18370031/
# 应用脚本
$ su -- root
$ sh /data/oraInventory/orainstRoot.sh
$ sh /data/app/11.2.0/grid/root.sh
4、校验服务(所有节点执行)
bash
[root@rac1 bin]# ./crs_stat -t -v
Name Type R/RA F/FT Target State Host
----------------------------------------------------------------------
ora....ER.lsnr ora....er.type 0/5 0/ ONLINE ONLINE rac1
ora....N1.lsnr ora....er.type 0/5 0/0 ONLINE ONLINE rac1
ora.OCR.dg ora....up.type 0/5 0/ ONLINE ONLINE rac1
ora.asm ora.asm.type 0/5 0/ ONLINE ONLINE rac1
ora.cvu ora.cvu.type 0/5 0/0 ONLINE ONLINE rac1
ora.gsd ora.gsd.type 0/5 0/ OFFLINE OFFLINE
ora....SM1.asm application 0/5 0/0 ONLINE ONLINE rac1
ora....ER.lsnr application 0/5 0/0 ONLINE ONLINE rac1
ora.rac1.gsd application 0/5 0/0 OFFLINE OFFLINE
ora.rac1.ons application 0/3 0/0 ONLINE ONLINE rac1
ora.rac1.vip ora....t1.type 0/0 0/0 ONLINE ONLINE rac1
ora....network ora....rk.type 0/5 0/ ONLINE ONLINE rac1
ora.oc4j ora.oc4j.type 0/1 0/2 ONLINE ONLINE rac1
ora.ons ora.ons.type 0/3 0/ ONLINE ONLINE rac1
ora.scan1.vip ora....ip.type 0/0 0/0 ONLINE ONLINE rac1
ora....SM2.asm application 0/5 0/0 ONLINE ONLINE rac2
ora....BY.lsnr application 0/5 0/0 ONLINE ONLINE rac2
ora....dby.gsd application 0/5 0/0 OFFLINE OFFLINE
ora....dby.ons application 0/3 0/0 ONLINE ONLINE rac2
ora....dby.vip ora....t1.type 0/0 0/0 ONLINE ONLINE rac2
三、安装Oracle软件
1、执行安装(任一节点)
bash
$ su -- oracle
$ cd /data/Download/software/database/
$ export DISPLAY=[]:0.0
$ ./ runInstaller

四、ASM配置
1、执行安装(任一节点)
注意:关于ASM磁盘划分,主要分为裁决盘、数据盘、归档盘,其中裁决盘三块盘组成,否则会报错
bash
$ su -- grid
$ export DISPLAY=[]:0.0
$ asmca
五、建库
1、开始建库(任一节点)
bash
$ su -- oracle
$ export DISPLAY=[]:0.0
$ dbca
2、检查数据库状态
bash
$ su -- oracle
$ srcvtl status database --d rac
Instance rac1 is running on node master
Instance rac2 is running on node standby