一、安装环境设置

1.1 硬件环境

1.2 软件环境

1.2.1 安装python

 -- root用户【新增备库节点】  

# 本次选择安装python 3.6.5版本
mkdir /usr/local/python3
cd /usr/local/python3
wget https://www.python.org/ftp/python/3.6.5/Python-3.6.5.tgz
tar -zxf Python-3.6.5.tgz
cd /usr/local/python3/Python-3.6.5
./configure --prefix=/usr/local/python3 --enable-shared CFLAGS=-fPIC
make && make install
ln -s /usr/local/python3/bin/python3 /usr/bin/python3
ln -s /usr/local/python3/bin/pip3 /usr/bin/pip3
​
-- 设置环境变量
cat >>/etc/profile<> /etc/ld.so.conf
ldconfig
​
-- 如遇如下报错
python3: error while loading shared libraries: libpython3.6m.so.1.0: cannot open shared object file: No such file or directory
可执行 cp /usr/local/python3/Python-3.6.5/libpython3.6m.so.1.0 /usr/lib64/

1.2.2 安装软件依赖包

1.2.2.1 软件依赖要求

1.2.2.2 安装软件依赖包

 -- root用户 【新增备库节点】  

-- 安装依赖包
yum install -y libaio-devel flex bison ncurses-devel glibc-devel patch redhat-lsb-core readline-devel zlib readline gcc perl-ExtUtils-Embed readline-devel zlib-devel expect
​
-- 检查是否已安装
rpm -qa --queryformat "%{NAME}-%{VERSION}-%{RELEASE} (%{ARCH})\n" | grep -E "libaio-devel|flex|bison|ncurses-devel|glibc-devel|patch|redhat-lsb-core|readline-devel|zlib|readline|gcc|python|python-devel|perl-ExtUtils-Embed|readline-devel|zlib-devel|expect"

1.2.3 操作系统配置

1.2.3.1 修改操作系统参数

 -- root用户 【新增备库节点】  

-- 添加如下参数：
cat>>/etc/sysctl.conf <> /etc/rc.d/rc.local< /sys/kernel/mm/transparent_hugepage/enabled
fi
if test -f /sys/kernel/mm/transparent_hugepage/defrag;

 then  
     echo never > /sys/kernel/mm/transparent_hugepage/defrag  

fi
EOF
​
-- 查看是否关闭：
cat /sys/kernel/mm/transparent_hugepage/enabled
cat /sys/kernel/mm/transparent_hugepage/defrag

1.2.3.3 关闭防火墙

 -- root用户 【新增备库节点】  

systemctl disable firewalld.service
systemctl stop firewalld.service

1.2.3.4 关闭selinux

 -- root用户 【新增备库节点】  

sed -i "s/SELINUX=enforcing/SELINUX=disabled/" /etc/selinux/config
-- 检查
cat /etc/selinux/config | grep disabled

1.2.3.5 修改字符集

 -- root用户 【新增备库节点】  

cat>> /etc/profile<>/etc/ssh/sshd_config<>/etc/ssh/sshd_config<>/etc/resolv.conf<> /etc/hosts<

 -- root用户 【新增备库节点】  

-- 创建dbgrp用户组,组ID同主库
[root@opengauss-db4 ~]# /usr/sbin/groupadd -g 1004 dbgrp
-- 创建omm用户，用户ID同主库
[root@opengauss-db4 ~]# /usr/sbin/useradd -u 1003 -g dbgrp -G dbgrp omm
-- 设置omm用户密码
[root@opengauss-db4 ~]# echo "omm123" | passwd --stdin omm

3.1.2 配置互信

 -- 配置互信，配置root用户及omm用户互信  

​
-- 配置 root 用户互信【主节点 root用户执行】
[root@opengauss-db1 ~]# cd /opt/software/openGauss/script/
[root@opengauss-db1 script]# vim hostfile -- 在该目录下编辑所有节点IP文件，参照如下
192.168.17.113
192.168.17.139
192.168.17.159
192.168.17.112
-- 执行gs_sshexkey 配置互信
[root@opengauss-db1 script]# ./gs_sshexkey -f hostfile
Please enter password for current user[root].
Password: -- 输入 root口令
Checking network information.
All nodes in the network are Normal.
Successfully checked network information.
Creating SSH trust.
Creating the local key file.
Successfully created the local key files.
Appending local ID to authorized_keys.
Successfully appended local ID to authorized_keys.
Updating the known_hosts file.
Successfully updated the known_hosts file.
Appending authorized_key on the remote node.
Successfully appended authorized_key on all remote node.
Checking common authentication file content.
Successfully checked common authentication content.
Distributing SSH trust file to all node.
Distributing trust keys file to all node successfully.
Successfully distributed SSH trust file to all node.
Verifying SSH trust on all hosts.
Successfully verified SSH trust on all hosts.
Successfully created SSH trust.
​
-- 测试root用户互信
[root@opengauss-db1 script]# ssh 192.168.17.112
[root@opengauss-db1 script]# ssh 192.168.17.139
[root@opengauss-db1 script]# ssh 192.168.17.159
​
-- 配置 omm 用户互信 【主节点 omm 用户执行】
[omm@opengauss-db1 ~]$ cd /u01/app/software/script/
[omm@opengauss-db1 script]$ ./gs_sshexkey -f hostfile
Please enter password for current user[omm].
Password:
Checking network information.
All nodes in the network are Normal.
Successfully checked network information.
Creating SSH trust.
Creating the local key file.
Successfully created the local key files.
Appending local ID to authorized_keys.
Successfully appended local ID to authorized_keys.
Updating the known_hosts file.
Successfully updated the known_hosts file.
Appending authorized_key on the remote node.
Successfully appended authorized_key on all remote node.
Checking common authentication file content.
Successfully checked common authentication content.
Distributing SSH trust file to all node.
Distributing trust keys file to all node successfully.
Successfully distributed SSH trust file to all node.
Verifying SSH trust on all hosts.
Successfully verified SSH trust on all hosts.
Successfully created SSH trust.
​
-- 测试omm用户互信
[omm@opengauss-db1 script]$ ssh 192.168.17.112
[omm@opengauss-db1 script]$ ssh 192.168.17.139
[omm@opengauss-db1 script]$ ssh 192.168.17.159
​
-- gs_sshexkey 详细语法可查看 gs_sshexkey -? | --help

3.1.3 配置新增节点环境变量

 -- 拷贝已有节点omm用户环境变量文件至新增节点  

[omm@opengauss-db1 ~]$ scp .bashrc omm@192.168.17.112:/home/omm/

[omm@opengauss-db1 ~]$ scp .bash_profile omm@192.168.17.112:/home/omm/
​
-- 修改新增节点 omm用户.bashrc, 删除 或 注释 export GAUSS_ENV=2
# GAUSS_ENV 值为2表示安装成功
​
-- 生效新增节点 omm 用户环境变量
[omm@opengauss-db4 ~]$ source .bashrc
[omm@opengauss-db4 ~]$ source .bash_profil
​
-- 拷贝已有节点root用户环境变量文件至新增节点
[root@opengauss-db1 ~]# scp -r /etc/profile root@192.168.17.112:/etc/profile
​
-- 生效新增节点root用户环境变量
[root@opengauss-db4 ~]# source /etc/profile

3.1.4 修改XML配置文件

3.1.4.1 配置XML文件

 -- 主节点 root 用户  

-- 在 /opt/software/openGauss 目录下修改 clusterconfig.xml配置文件
-- 执行操作如下
cat > clusterconfig.xml<

​

​

EOF

3.1.4.2 参数简介

3.1.5 拷贝主节点clusterconfig.xml至新增节点

 -- 新增节点创建 /opt/software/openGauss 目录  

[root@opengauss-db4 ]# mkdir -p /opt/software/openGauss
-- 拷贝主节点 clusterconfig.xml 文件至新增节点同目录处
[root@opengauss-db1 ]# scp /opt/software/openGauss/clusterconfig.xml omm@opengauss-db4:/opt/software/openGauss

3.2 集群扩容

 -- 主节点 【root 用户执行】  

[omm@opengauss-db1 ~]$ su - root
Password:
Last login: Fri Nov 11 15:14:54 CST 2022 from 192.168.16.10 on pts/0
[root@opengauss-db1 ~]# cd /opt/software/openGauss/script
[root@opengauss-db1 script]# source /home/omm/.bashrc
[root@opengauss-db1 script]# ./gs_expansion -U omm -G dbgrp -X /opt/software/openGauss/clusterconfig.xml -h 192.168.17.112
​
执行结果如下：
Start expansion with cluster manager component.
Start to send soft to each standby nodes.
End to send soft to each standby nodes.
Success to send XML to new nodes
Start to perform perinstall on nodes: ['opengauss-db4']
Preinstall command is: /tmp/gs_expansion_2022-11-13_21_31_49_537605/pkg/script/gs_preinstall -U omm -G dbgrp -X /opt/software/openGauss/clusterconfig.xml -L --non-interactive 2>&1
Success to perform perinstall on nodes ['opengauss-db4']
Success to change user to [omm]
Installing applications on all new nodes.
Install on new node output: [SUCCESS] opengauss-db4:
Using omm:dbgrp to install database.
Using installation program path : /opt/gaussdb/install/app
Command for creating symbolic link: ln -snf /opt/gaussdb/install/app_4e931f9a /opt/gaussdb/install/app.
Decompressing bin file.
Decompress CM package command: export LD_LIBRARY_PATH=$GPHOME/script/gspylib/clib:$LD_LIBRARY_PATH && tar -zxf "/opt/gaussdb/gausstools/om/script/os_platform/./../../openGauss-3.1.0-CentOS-64bit-cm.tar.gz" -C "/opt/gaussdb/install/app"
Decompress CM package successfully.
Successfully decompressed bin file.
Modifying Alarm configuration.
Modifying user's environmental variable $GAUSS_ENV.
Successfully modified user's environmental variable $GAUSS_ENV.
Fixing file permission.
Set Cgroup config file to appPath.
Successfully Set Cgroup.
​
Successfully installed APP on nodes ['opengauss-db4'].
success to send all CA file.
Success to change user to [omm]
Success to init instance on nodes ['opengauss-db4']
Start to generate and send cluster static file.
End to generate and send cluster static file.
​
Ready to perform command on node [opengauss-db4]. Command is : source /home/omm/.bashrc;gs_guc set -D /opt/gaussdb/install/data/db1 -h 'host all omm 192.168.17.112/32 trust' -h 'host all all 192.168.17.112/32 sha256'
Successfully set hba on all nodes.
Success to change user to [omm]

Stopping cluster.

Successfully stopped cluster.

End stop cluster.
Remove dynamic_config_file and CM metadata directory on all nodes.

Starting cluster.

Successfully started primary instance. Wait for standby instance.

.

Successfully started cluster.

cluster_state : Normal
redistributing : No
node_count : 4
Datanode State

 primary           : 1  
 standby           : 3  
 secondary         : 0  
 cascade_standby   : 0  
 building          : 0  
 abnormal          : 0  
 down              : 0  

​
Expansion results:
192.168.17.112: Success
​
-- 执行过程日志可查看 /opt/gaussdb/log/omm/pg_log/dn__xxx 目录下 postgresql 日志

3.3 查询集群状态

 [root@opengauss-db1 script]# su - omm  

Last login: Sun Nov 13 21:37:57 CST 2022 on pts/4
[omm@opengauss-db1 ~]$ cm_ctl query -v -C -i -d
[ CMServer State ]
​

node node_ip instance state

1 opengauss-db1 192.168.17.113 1 /opt/gaussdb/install/data/cm/cm_server Primary
2 opengauss-db2 192.168.17.139 2 /opt/gaussdb/install/data/cm/cm_server Standby
3 opengauss-db3 192.168.17.159 3 /opt/gaussdb/install/data/cm/cm_server Standby
4 opengauss-db4 192.168.17.112 4 /opt/gaussdb/install/data/cm/cm_server Standby
​
[ Cluster State ]
​
cluster_state : Normal
redistributing : No
balanced : Yes
current_az : AZ_ALL
​
[ Datanode State ]
​

node node_ip instance state | node node_ip instance state | node node_ip instance state | node node_ip instance state

1 opengauss-db1 192.168.17.113 6001 /opt/gaussdb/install/data/db1 P Primary Normal | 2 opengauss-db2 192.168.17.139 6002 /opt/gaussdb/install/data/db1 S Standby Normal | 3 opengauss-db3 192.168.17.159 6003 /opt/gaussdb/install/data/db1 S Standby Normal | 4 opengauss-db4 192.168.17.112 6004 /opt/gaussdb/install/data/db1 S Standby Normal
​
​
[omm@opengauss-db1 ~]$ gs_om -t status --detail
[ CMServer State ]
​

node node_ip instance state

1 opengauss-db1 192.168.17.113 1 /opt/gaussdb/install/data/cm/cm_server Primary
2 opengauss-db2 192.168.17.139 2 /opt/gaussdb/install/data/cm/cm_server Standby
3 opengauss-db3 192.168.17.159 3 /opt/gaussdb/install/data/cm/cm_server Standby
4 opengauss-db4 192.168.17.112 4 /opt/gaussdb/install/data/cm/cm_server Standby
​
[ Cluster State ]
​
cluster_state : Normal
redistributing : No
balanced : Yes
current_az : AZ_ALL
​
[ Datanode State ]
​

node node_ip instance state

1 opengauss-db1 192.168.17.113 6001 /opt/gaussdb/install/data/db1 P Primary Normal
2 opengauss-db2 192.168.17.139 6002 /opt/gaussdb/install/data/db1 S Standby Normal
3 opengauss-db3 192.168.17.159 6003 /opt/gaussdb/install/data/db1 S Standby Normal
4 opengauss-db4 192.168.17.112 6004 /opt/gaussdb/install/data/db1 S Standby Normal

3.4 刷新动态配置文件

 -- 主节点 omm 用户操作  

[omm@opengauss-db1 ~]$ gs_om -t refreshconf
Generating dynamic configuration file for all nodes.
Successfully generated dynamic configuration file.

3.5 数据测试

 -- 主节点  

[root@opengauss-db1 ~]# su - omm
Last login: Mon Nov 14 10:14:24 CST 2022 on pts/5
[omm@opengauss-db1 ~]$ gsql -d postgres -p 26000
gsql ((openGauss 3.1.0 build 4e931f9a) compiled at 2022-09-29 14:19:24 commit 0 last mr )
Non-SSL connection (SSL connection is recommended when requiring high-security)
Type "help" for help.
​
openGauss=# create table pgtb (id int, name text);
CREATE TABLE
openGauss=# insert into pgtb (id,name) values (10,'Jacky');
INSERT 0 1
openGauss=#
​
-- 新增节点
[omm@opengauss-db4 ~]$ gsql -d postgres -p 26000
gsql ((openGauss 3.1.0 build 4e931f9a) compiled at 2022-09-29 14:19:24 commit 0 last mr )
Non-SSL connection (SSL connection is recommended when requiring high-security)
Type "help" for help.
​
openGauss=# select * from pgtb;

(1 row)
​
openGauss=# insert into pgtb (id,name) values (11,'shlei6067');
ERROR: cannot execute INSERT in a read-only transaction
openGauss=#

四、补充

4.1 扩容时报GAUSS-51100

 --问题现象  

主节点执行扩容时，报 [GAUSS-51100] : Failed to verify SSH trust on these nodes错误
[root@opengauss-db1 openGauss]# ./script/gs_expansion -U omm -G dbgrp -X /opt/software/openGauss/clusterconfig.xml -h 192.168.17.112
[GAUSS-51100] : Failed to verify SSH trust on these nodes:
opengauss-db1, opengauss-db2, opengauss-db3, opengauss-db4, 192.168.17.113, 192.168.17.139, 192.168.17.159, 192.168.17.112 by root
opengauss-db1, opengauss-db2, opengauss-db3, opengauss-db4, 192.168.17.113, 192.168.17.139, 192.168.17.159, 192.168.17.112 by individual user.
​
-- 解决办法
创建存放所有节点IP文件，分别在root用户及omm用户下参照3.1.2章节执行 gs_sshexkey 来配置节点间的互信，并分别测试互信
可在主节点执行 /opt/gaussdb/gausstools/om/script/gspylib/pssh/bin/pssh -s -H 'pwd' 查看互信
​
# gs_expansion文件倒数第二行expansion.checkTrust()是用来做互信校验的

4.2 解析clusterconfig.xml报错

 -- 问题现象：  

主节点执行扩容时报如下错误：
[root@opengauss-db1 script]# ./gs_expansion -U omm -G dbgrp -X /opt/software/openGauss/clusterconfig.xml -h 192.168.17.112
Start expansion with cluster manager component.
Start to send soft to each standby nodes.
End to send soft to each standby nodes.
Success to send XML to new nodes
Start to perform perinstall on nodes: ['opengauss-db4']
Preinstall command is: /tmp/gs_expansion_2022-11-13_21_21_21_185901/pkg/script/gs_preinstall -U omm -G dbgrp -X /opt/software/openGauss/clusterconfig.xml -L --non-interactive 2>&1
Success to perform perinstall on nodes ['opengauss-db4']
Success to change user to [omm]
Installing applications on all new nodes.
Install on new node output: [FAILURE] opengauss-db4:
Using omm:dbgrp to install database.
Using installation program path : /opt/gaussdb/install/app
Command for creating symbolic link: ln -snf /opt/gaussdb/install/app_4e931f9a /opt/gaussdb/install/app.
[GAUSS-51234] : The configuration file [/opt/software/openGauss/clusterconfig.xml] contains parsing errors. Error:
[GAUSS-51236] : Failed to parsing xml. Error:
[Errno 13] Permission denied: '/opt/software/openGauss/clusterconfig.xml'.
​
Process Process-1:
Traceback (most recent call last):
File "/usr/local/python3/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap

 self.run()  

File "/usr/local/python3/lib/python3.6/multiprocessing/process.py", line 93, in run

 self._target(*self._args, **self._kwargs)  

File "/opt/software/openGauss/script/impl/expansion/expansion_impl_with_cm.py", line 446, in do_install

 self.install_app()  

File "/opt/software/openGauss/script/impl/expansion/expansion_impl_with_cm.py", line 172, in install_app

 [key for key in result_map.keys() if result_map[key] == "Failure"])  

Exception: [GAUSS-52707] : Failed to install on ['opengauss-db4'].
​
-- 解决办法：
可将主节点clusterconfig.xml文件拷贝至新增备库相同目录节点下，并赋予omm用户读权限，参照 3.1.5 章节
该问题主要是由于 3.1.0 版本在扩容时拷贝文件到新增节点，clusterconfig.xml 用户和属组都是 root，omm用户无法读该文件

4.3 扩容时报GAUSS-51802

 -- 问题现象：  

[GAUSS-51802] : Failed to obtain the environment variable "GAUSSHOME", please import environment variable.
​
-- 解决办法：
主节点root用户执行 source /home/omm/.bashrc
新增节点 omm 用户执行 source /home/omm/.bashrc

4.4 扩容时报GAUSS-51800

 -- 问题现象：  

扩容时，报如下错误：
[root@opengauss-db1 script]# source /home/omm/.bashrc
[root@opengauss-db1 script]# ./script/gs_expansion -U omm -G dbgrp -X /opt/software/openGauss/clusterconfig.xml -h 192.168.17.112
-bash: ./script/gs_expansion: No such file or directory
[root@opengauss-db1 script]# ./gs_expansion -U omm -G dbgrp -X /opt/software/openGauss/clusterconfig.xml -h 192.168.17.112
Start expansion with cluster manager component.
Start to send soft to each standby nodes.
End to send soft to each standby nodes.
Success to send XML to new nodes
Start to perform perinstall on nodes: ['opengauss-db4']
Preinstall command is: /tmp/gs_expansion_2022-11-13_20_49_42_523008/pkg/script/gs_preinstall -U omm -G dbgrp -X /opt/software/openGauss/clusterconfig.xml -L --non-interactive 2>&1
Success to perform perinstall on nodes ['opengauss-db4']
Success to change user to [omm]
Installing applications on all new nodes.
Install on new node output: [FAILURE] opengauss-db4:
Using omm:dbgrp to install database.
Using installation program path : /opt/gaussdb/install/app
[GAUSS-51800] : The environmental variable $GAUSSHOME is empty. or variable has exceeded maximum length
​
Process Process-1:
Traceback (most recent call last):
File "/usr/local/python3/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap

 self.run()  

File "/usr/local/python3/lib/python3.6/multiprocessing/process.py", line 93, in run

 self._target(*self._args, **self._kwargs)  

File "/opt/software/openGauss/script/impl/expansion/expansion_impl_with_cm.py", line 446, in do_install

 self.install_app()  

File "/opt/software/openGauss/script/impl/expansion/expansion_impl_with_cm.py", line 172, in install_app

 [key for key in result_map.keys() if result_map[key] == "Failure"])  

Exception: [GAUSS-52707] : Failed to install on ['opengauss-db4'].
​
-- 解决办法：
新增节点 omm 用户 修改 /home/omm/.bashrc ,取消或注释 GAUSS_ENV
然后执行 source /home/omm/.bashrc 生效环境变量

4.5 主节点/opt/gaussdb/gausstools/ 目录为空

 -- 问题现象：  

在扩容时，发现主节点 /opt/gaussdb/gausstools/ 目录为空
​
-- 解决办法：
使用 omm 用户 拷贝其它正常备库节点 /opt/gaussdb/gausstools/ 目录至 主节点 omm用户 /opt/gaussdb/gausstools/ 下