一、背景
大数据cdh，EC的数据库
最终架构
二、master原始配置信息
- IP信息
makefile
#
172.25.189.7:3306
root/Ec@0828- my.cnf文件
ini
vim /etc/my.cnf
# For advice on how to change settings please see
# http://dev.mysql.com/doc/refman/5.7/en/server-configuration-defaults.html
[mysqld]
#
# Remove leading # and set to the amount of RAM for the most important data
# cache in MySQL. Start at 70% of total RAM for dedicated server, else 10%.
# innodb_buffer_pool_size = 128M
#
# Remove leading # to turn on a very important data integrity option: logging
# changes to the binary log between backups.
# log_bin
log_bin=/opt/mysql/mysql-bin
server_id=2
binlog_format = row
# Remove leading # to set options mainly useful for reporting servers.
# The server defaults are faster for transactions and fast SELECTs.
# Adjust sizes as needed, experiment to find the optimal values.
# join_buffer_size = 128M
# sort_buffer_size = 2M
# read_rnd_buffer_size = 2M
datadir=/opt/mysql/data/
socket=/var/lib/mysql/mysql.sock
# Disabling symbolic-links is recommended to prevent assorted security risks
symbolic-links=0
log-error=/var/log/mysqld.log
pid-file=/var/run/mysqld/mysqld.pid
character_set_server=utf8- systemd信息
ini
vim /etc/systemd/system/multi-user.target.wants/mysqld.service
# Copyright (c) 2015, 2016, Oracle and/or its affiliates. All rights reserved.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
#
# systemd service file for MySQL forking server
#
[Unit]
Description=MySQL Server
Documentation=man:mysqld(8)
Documentation=http://dev.mysql.com/doc/refman/en/using-systemd.html
After=network.target
After=syslog.target
[Install]
WantedBy=multi-user.target
[Service]
User=mysql
Group=mysql
Type=forking
PIDFile=/var/run/mysqld/mysqld.pid
# Disable service start and stop timeout logic of systemd for mysqld service.
TimeoutSec=0
# Execute pre and post scripts as root
PermissionsStartOnly=true
# Needed to create system tables
ExecStartPre=/usr/bin/mysqld_pre_systemd
# Start main service
ExecStart=/usr/sbin/mysqld --daemonize --pid-file=/var/run/mysqld/mysqld.pid $MYSQLD_OPTS
# Use this to switch malloc implementation
EnvironmentFile=-/etc/sysconfig/mysql
# Sets open_files_limit
LimitNOFILE = 5000
Restart=on-failure
RestartPreventExitStatus=1
PrivateTmp=false- 启停命令
lua
systemctl status mysqld- 相关配置信息
sql
mysql> show variables like '%innodb_buffer_pool_size%';
+-------------------------+-----------+
| Variable_name | Value |
+-------------------------+-----------+
| innodb_buffer_pool_size | 134217728 |
+-------------------------+-----------+
1 row in set (0.01 sec) - binlog配置信息
sql
mysql> show variables like '%binlog%';
+--------------------------------------------+----------------------+
| Variable_name | Value |
+--------------------------------------------+----------------------+
| binlog_cache_size | 32768 |
| binlog_checksum | CRC32 |
| binlog_direct_non_transactional_updates | OFF |
| binlog_error_action | ABORT_SERVER |
| binlog_format | ROW |
| binlog_group_commit_sync_delay | 0 |
| binlog_group_commit_sync_no_delay_count | 0 |
| binlog_gtid_simple_recovery | ON |
| binlog_max_flush_queue_time | 0 |
| binlog_order_commits | ON |
| binlog_row_image | FULL |
| binlog_rows_query_log_events | OFF |
| binlog_stmt_cache_size | 32768 |
| binlog_transaction_dependency_history_size | 25000 |
| binlog_transaction_dependency_tracking | COMMIT_ORDER |
| innodb_api_enable_binlog | OFF |
| innodb_locks_unsafe_for_binlog | OFF |
| log_statements_unsafe_for_binlog | ON |
| max_binlog_cache_size | 18446744073709547520 |
| max_binlog_size | 1073741824 |
| max_binlog_stmt_cache_size | 18446744073709547520 |
| sync_binlog | 1 |
+--------------------------------------------+----------------------+
22 rows in set (0.00 sec)
# binlog路径
# binlog大小
# binlog保留时间
# 三、172.25.189.8从库部署【完成】
- mysql安装
bash
cd /usr/local
wget https://cdn.mysql.com/archives/mysql-5.7/mysql-5.7.27-linux-glibc2.12-x86_64.tar.gz
tar -zxvf mysql-5.7.27-linux-glibc2.12-x86_64.tar.gz
ln -s mysql-5.7.27-linux-glibc2.12-x86_64 mysql
groupadd mysql
useradd -r -g mysql mysql
mkdir -p /opt/mysql/data
cd /opt/mysql
chown -R mysql:mysql /opt/mysql
./bin/mysqld --initialize --user=mysql --datadir=/opt/mysql/data
# HX4m*Ztor7Tw
# ./bin/mysqld_safe --datadir=/opt/mysql/data &
# master
mysql -h172.25.189.7 -uroot -p'Ec@0828'
# slave
mysql -h172.25.189.8 -uroot -p'Ec@0828'
# 重置local root密码
ALTER USER 'root'@'localhost' IDENTIFIED BY '123';
FLUSH PRIVILEGES;
EXIT;
# 创建 % root的密码
create user 'root'@'%' identified by 'Ec@0828';
grant all on *.* to root with grant option;
flush privileges;
show grants for 'root'@'%';
#
SHOW VARIABLES LIKE '%read_only';- my.cnf 配置
ini
cat > /etc/my.cnf << EOF
[mysqld]
super_read_only = ON
report-host=172.25.189.8
user = mysql
port = 3306
# 主从复制配置
server_id=3
relay-log=mysql-relay-bin
log-slave-updates=ON
gtid-mode=ON
enforce-gtid-consistency=ON
slave-parallel-type=LOGICAL_CLOCK
slave-parallel-workers=4
slave_preserve_commit_order=on
log_bin=/opt/mysql/mysql-bin
binlog-format = row
expire_logs_days=7
max_binlog_size = 500M
datadir=/opt/mysql/data/
socket=/var/lib/mysql/mysql.sock
symbolic-links=0
log-error=/var/log/mysqld.log
pid-file=/var/run/mysqld/mysqld.pid
character_set_server=utf8
EOF- systemd配置
ini
cat > /etc/systemd/system/mysqld.service << EOF
[Unit]
Description=MySQL 5.7 Database Server
After=network.target
[Service]
User=mysql
Group=mysql
ExecStart=/usr/local/mysql/bin/mysqld --defaults-file=/etc/my.cnf
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF- 启动mysql
bash
systemctl daemon-reload && systemctl start mysqld
# 配置环境变量
echo 'export PATH="/usr/local/mysql/bin:$PATH"' >> /etc/profile.d/mysql.sh
source /etc/profile- 配置主从信息
如下
四、keepalived配置【完成】
-
master上安装keepalived 使用yum方式安装
-
master上配置keepalived
bash
vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
vrrp_script chk_haproxy {
script "/etc/keepalived/check_master_vip.sh"
interval 2
}
#router_id需要根据具体业务命名，以便于不同套作区分，同套keepalived的2台此参数名相同
global_defs {
router_id ks_pqsql
}
vrrp_instance VI_1 {
state BACKUP
#网口名称需要确认是否为ens192，如果不是需要修改为对应网口信息
interface ens192
#virtual_router_id 取值范围为0~255，同vip网段需要确认不能和其他套keepalived共用一个值，同套keepalived的2台此参数名相同
virtual_router_id 245
#priority取值自定义，区别2台keepalived的权重，一台值偏小
priority 100
advert_int 1
nopreempt
#auth_pass需要一套环境配置为相同信息
authentication {
auth_type PASS
auth_pass Yumc1234
}
track_script {
# chk_haproxy
}
#172.xx.xx.xx/xx需要修改为真实vip的网段ip信息，网口名称需要确认是否为ens192，如果不是需要修改为对应网口信息
virtual_ipaddress {
172.25.191.239/22 dev ens192
}
#172.25.89.xx改为实际vip地址，172.25.88.1改为实际网关地址
notify_master "/usr/sbin/arping -I ens192 -c 2 -s 172.25.191.239 172.25.188.1"
}注意： 1、script "/etc/keepalived/check_master_vip.sh" 2、interface ens192 3、priority 100 4、172.25.191.239/22 dev ens192 5、notify_master "/usr/sbin/arping -I ens192 -c 2 -s 172.25.191.239 172.25.188.1"
-
slave上安装keepalived
-
slave上配置keepalived
bash
vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
vrrp_script chk_haproxy {
script "/etc/keepalived/check_master_vip.sh"
interval 2
}
#router_id需要根据具体业务命名，以便于不同套作区分，同套keepalived的2台此参数名相同
global_defs {
router_id ks_pqsql
}
vrrp_instance VI_1 {
state BACKUP
#网口名称需要确认是否为ens192，如果不是需要修改为对应网口信息
interface ens192
#virtual_router_id 取值范围为0~255，同vip网段需要确认不能和其他套keepalived共用一个值，同套keepalived的2台此参数名相同
virtual_router_id 245
#priority取值自定义，区别2台keepalived的权重，一台值偏小
priority 80
advert_int 1
nopreempt
#auth_pass需要一套环境配置为相同信息
authentication {
auth_type PASS
auth_pass Yumc1234
}
track_script {
# chk_haproxy
}
#172.xx.xx.xx/xx需要修改为真实vip的网段ip信息，网口名称需要确认是否为ens192，如果不是需要修改为对应网口信息
virtual_ipaddress {
172.25.191.239/22 dev ens192
}
#172.25.89.xx改为实际vip地址，172.25.88.1改为实际网关地址
notify_master "/usr/sbin/arping -I ens192 -c 2 -s 172.25.191.239 172.25.188.1"
}-
启动2个keepalived
-
检查VIP
-
完成
五、主从配置
- 备份master的全量数据
csharp
# 全备数据
#
# 创建复制账号
CREATE USER 'replica'@'%' identified WITH mysql_native_password by 'Asdf@1234';
# 复制账号权限配置
GRANT REPLICATION SLAVE ON *.* TO 'replica'@'%';
GRANT REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'replica'@'%';
# 刷新
flush privileges;- slave上恢复备份文件
ini
#
set global read_only=0;
#
source xxxx;
# 配置主从信息
CHANGE MASTER TO MASTER_HOST='172.21.27.30' ,MASTER_USER='root' ,MASTER_PASSWORD='Asdf@1234',master_auto_position=1;
# 启动slave，并启动复制
start slave;六、主库上调整点
-
安装keepalived 【完成】
-
my.cnf 修改
- binlog保留时间为7天
- binlog文件大小500MB滚动
- innodb_buffer_pool_size 写进配置文件
ini
vim /etc/my.cnf
[mysqld]
super_read_only = ON
report-host=172.25.189.7
user = mysql
port = 3306
# 主从复制配置
server_id=2
relay-log=mysql-relay-bin
log-slave-updates=ON
gtid-mode=ON
enforce-gtid-consistency=ON
slave-parallel-type=LOGICAL_CLOCK
slave-parallel-workers=4
slave_preserve_commit_order=on
log_bin=/opt/mysql/mysql-bin
binlog-format = row
expire_logs_days=7
max_binlog_size = 500M
# Remove leading # to set options mainly useful for reporting servers.
# The server defaults are faster for transactions and fast SELECTs.
# Adjust sizes as needed, experiment to find the optimal values.
# join_buffer_size = 128M
# sort_buffer_size = 2M
# read_rnd_buffer_size = 2M
datadir=/opt/mysql/data/
socket=/var/lib/mysql/mysql.sock
# Disabling symbolic-links is recommended to prevent assorted security risks
symbolic-links=0
log-error=/var/log/mysqld.log
pid-file=/var/run/mysqld/mysqld.pid
character_set_server=utf8- 重启主库mysqld
arduino
#
systemctl restart mysqld七、切换与上线
- 主库上参数调整，重启
内容见：上方 6.2
- 主库上数据备份
javascript
# 备份文件在 172.21.240.67:/data/package/ec_cdh 下
#
# mysqldump -h172.25.189.7 -uroot -p'Ec@0828' --single-transaction --flush-logs metastore > metastore_backup250217.sql
#
# mysqldump -h172.25.189.7 -uroot -p'Ec@0828' --flush-logs scm > /data/package/ec_cdh/scm_backup250217.sql
# 备份所有的库，防止主从复制出现元数据的问题
mysqldump -h172.25.189.7 -uroot -p'Ec@0828' --single-transaction --all-databases --flush-logs > /data/package/ec_cdh/all_backup250218.sql- 从库上数据恢复
python
# 从库
mysql -h172.25.189.8 -uroot -p'Ec@0828'
#
SHOW GLOBAL VARIABLES LIKE '%read_only';
#
set global read_only =0;
source all_backup250218.sql;
# use scm;
# source scm_backup250217.sql;
# use metastore;
# source metastore_backup250217.sql;
# 开启 只读
set global super_read_only =1;
# 库大小
select TABLE_SCHEMA, concat(round(sum(DATA_LENGTH/1024/1024),2),'MB') as dataMB from information_schema.TABLES GROUP BY TABLE_SCHEMA;
# 查询表大小
select
table_schema as '数据库',
sum(table_rows) as '记录数',
sum(truncate(data_length/1024/1024, 2)) as '数据容量(MB)',
sum(truncate(index_length/1024/1024, 2)) as '索引容量(MB)'
from information_schema.tables
group by table_schema
order by sum(data_length) desc, sum(index_length) desc;- 主从关系建立
ini
# 配置slave
CHANGE MASTER TO MASTER_HOST='172.25.189.7' ,MASTER_USER='replica' ,MASTER_PASSWORD='1234',master_auto_position=1;
CHANGE MASTER TO MASTER_HOST='172.25.189.7' ,MASTER_USER='root' ,MASTER_PASSWORD='Ec@0828',master_auto_position=1;
# 开始
start slave;
#
show slave status\G
#
# 主库
mysql -h172.25.189.7 -uroot -p'Ec@0828' --ssl-mode=DISABLED
#
show slave hosts\G-
观察复制状态
-
主从自动检测与切换
直接执行sh check_mysql_master.sh，可以放Jenkins中执行，执行频率 30s 一次（可以适当缩短间隔时间）。
工具地址：http://172.21.141.48:8080/job/ElasticSearch/job/check.ECcdh.mysql.master/
vim check_mysql_master.sh
bash
#!/bin/bash
# 定义主从库的地址和端口
MASTER_HOST="172.25.189.7"
MASTER_PORT="3306"
SLAVE_HOST="172.25.189.8"
SLAVE_PORT="3306"
time=$(date)
# 定义MySQL用户和密码
MYSQL_USER="root"
MYSQL_PASSWORD="Ec@0828"
# 定义日志文件
LOG_FILE="/var/log/mysql_failover.log"
# 检查MySQL服务状态
check_mysql_status() {
local host=$1
local port=$2
local status
status=$(ssh $host systemctl is-active mysqld)
if [ "$status" == "active" ]; then
echo "MySQL service on $host is running."
return 0
else
echo "MySQL service on $host is not running."
return 1
fi
}
# 检查从库复制状态
check_slave_status() {
local host=$1
local port=$2
local status
status=$(mysql -h$host -P$port -u$MYSQL_USER -p$MYSQL_PASSWORD -e "SHOW SLAVE STATUS\G" | grep "Slave_IO_Running" | awk '{print $2}')
if [ "$status" == "Yes" ]; then
echo "Slave IO on $host is running."
return 0
else
echo "Slave IO on $host is not running."
return 1
fi
}
# 切换主从
promote_slave_to_master() {
local slave_host=$1
local slave_port=$2
local master_host=$3
local master_port=$4
echo "Promoting $slave_host to master..."
# 停止从库复制
mysql -h$slave_host -P$slave_port -u$MYSQL_USER -p$MYSQL_PASSWORD -e "STOP SLAVE;"
# 清除从库的复制信息
mysql -h$slave_host -P$slave_port -u$MYSQL_USER -p$MYSQL_PASSWORD -e "RESET SLAVE ALL;"
# 设置从库为只读模式
mysql -h$slave_host -P$slave_port -u$MYSQL_USER -p$MYSQL_PASSWORD -e "SET GLOBAL read_only = OFF;"
# 设置从库为新的主库
# mysql -h$slave_host -P$slave_port -u$MYSQL_USER -p$MYSQL_PASSWORD -e "CHANGE MASTER TO MASTER_HOST='$master_host', MASTER_PORT=$master_port, MASTER_USER='$MYSQL_USER', MASTER_PASSWORD='$MYSQL_PASSWORD', MASTER_AUTO_POSITION=1;"
echo "$slave_host has been promoted to master."
}
# 主函数
main() {
while true; do
# 检查主库状态
if ! check_mysql_status $MASTER_HOST; then
echo "$time Master MySQL service is down. Checking slave status..."
# 检查从库状态
if check_mysql_status $SLAVE_HOST && check_slave_status $SLAVE_HOST $SLAVE_PORT; then
echo "$time Slave MySQL service is running. Promoting slave to master..."
# 切换主从
promote_slave_to_master $SLAVE_HOST $SLAVE_PORT $MASTER_HOST $MASTER_PORT
# 更新主从地址
MASTER_HOST=$SLAVE_HOST
SLAVE_HOST="172.21.27.30"
echo "$time Failover completed. New master is $MASTER_HOST."
else
echo "$time Slave MySQL service is down. Cannot perform failover."
fi
else
echo "$time Master MySQL service is running."
fi
sleep 30
time=$(date)
done
}
# 执行主函数
main >> $LOG_FILE 2>&1- vip所在mysql是否主库判断
执行方式放进 keepalived的vrrp_script 模块中
vi /etc/keepalived/check_master_vip.sh
ini
#
#!/bin/bash
# MySQL连接信息
MYSQL_HOST="172.25.191.239"
MYSQL_USER="root"
MYSQL_PASSWORD="Ec@0828"
# 执行MySQL查询
output=$(mysql -h$MYSQL_HOST -u$MYSQL_USER -p$MYSQL_PASSWORD -e "SHOW GLOBAL VARIABLES LIKE '%read_only';" 2>/dev/null)
# 提取read_only和super_read_only的值
read_only=$(echo "$output" | awk '/read_only/ {print $2}')
super_read_only=$(echo "$output" | awk '/super_read_only/ {print $2}')
# 判断是否为从库
if [[ "$super_read_only" == "ON" ]]; then
echo "这是从库"
ssh -o "StrictHostKeyChecking=no" 172.25.191.239 "systemctl restart keepalived"
else
echo "这不是从库"
fi