使用python脚本部署k8s集群

1.环境规划：

节点	IP地址	操作系统	配置
脚本运行节点	192.168.174.5	centos7.9	2G2核
server	192.168.174.150	centos7.9	2G2核
client1	192.168.174.151	centos7.9	2G2核
client2	192.168.174.152	centos7.9	2G2

2.运行准备：

yum install -y python python-pip

pip install pexpect

3.完整脚本：

# coding=UTF-8
import sys,os,pexpect,subprocess,re

master_addresses=["192.168.174.150"]           # 主节点们的IP地址
master_domains=["server"]                          # 域名们
client_addresses=["192.168.174.151","192.168.174.152"]           # 从节点们的IP地址
client_domains=["client1","client2"]                          # 域名们

host_username="root"                                         # ssh连接的用户，控制端的用户为root
host_passwd="110119"                                         # ssh连接的用户密码

yum_mount_dev="/dev/cdrom"                                   # 本地yum源挂载设备路径
yum_mount_dir="/mnt/cdrom"                                   # 本地yum源挂载点

chrony_allows_addresses="192.168.174.0"                       # chrony主节点允许那个网段的从节点来同步时间

# ping主机
def ping_hosts(address):
	ping_flag=os.system("ping -c 3 {}".format(address))
	if ping_flag != 0:
		print("\033[1;31m" + "Failed:{address} ping失败" + "\033[0m")
		sys.exit("请检查!")
for i in range(0, len(master_addresses)):
	ping_hosts(master_addresses[i])
	print("\n")
for i in range(0, len(client_addresses)):
	ping_hosts(client_addresses[i])
	print("\n")


# 1.本地创建ssh公钥
if os.path.exists("/root/.ssh/id_rsa.pub") == True:
	print("\033[32m"+"ssh公钥已创建"+"\033[0m")                # 输出绿色字体
else:
	print("\033[32m"+"ssh公钥未创建，开始创建"+"\033[0m")
	child = pexpect.spawn('ssh-keygen -t rsa -b 1024')
	child.expect('Enter file in which to save the key')
	child.sendline('')
	child.expect('Enter passphrase')
	child.sendline('')
	child.expect('Enter same passphrase again')
	child.sendline('')

	child.expect(pexpect.EOF)               # 用于等待子进程的结束
	print(child.before.decode())            # 等待命令执行完毕并打印输出信息
	print("\033[32m" + "ssh公钥已创建" + "\033[0m")
	print("\n")


# 向被控主机添加公钥的方法
def add_ssh_public_key_client(address,username,password):
	print("\033[32m"+"{}正在被添加公钥".format(address)+"\033[0m")
	# BatchMode=yes：表示使SSH在连接过程中不会提示输入密码，而直接尝试免密连接，-o ConnectTimeout=5：表示限制连接超时时间为5秒
	public_key_flag=os.system("ssh {}@{} -o BatchMode=yes 'exit' &> /dev/null".format(username,address))
	if public_key_flag== 0:
		print("\033[32m" + "{}已经可以ssh连接".format(address) + "\033[0m")
		return
	child = pexpect.spawn('ssh-copy-id -i /root/.ssh/id_rsa.pub {}@{}'.format(username,address))
	try:
		child.expect('Are you sure you want to continue connecting')
	except pexpect.exceptions.TIMEOUT:       # 如果try块中的咨询超时5秒没有出现就会出现异常pexpect.TIMEOUT
		print("\033[32m"+"{}已经不是首次ssh连接了".format(address)+"\033[0m")
	else:                         # 是否回答咨询yes
		child.sendline('yes')
	finally:
		child.expect('password')
		child.sendline(password)
	child.expect(pexpect.EOF)               # 用于等待子进程的结束
	print(child.before.decode())            # 等待命令执行完毕并打印输出信息
# 测试ssh连接的方法
def test_ssh_connection(all_flag,address,username):
	print("\033[32m" + "{}测试是否可以ssh连接".format(address) + "\033[0m")
	flag=os.system('ssh {}@{} -o ConnectTimeout=5 "exit"'.format(username,address))
	if flag==0:
		print("\033[32m" + "Success: {}可以ssh免密连接".format(address) + "\033[0m")
	else:
		print("\033[1;31m" + "Failed: {}ssh免密连接失败".format(address) + "\033[0m")     # 输出红色字体
		all_flag=1
	return all_flag


# 本地的密钥开始加入被控制主机
for i in range(0, len(master_addresses)):
	add_ssh_public_key_client(master_addresses[i],host_username,host_passwd)
	print("\n")
for i in range(0, len(client_addresses)):
	add_ssh_public_key_client(client_addresses[i],host_username,host_passwd)
	print("\n")
# 测试ssh连接
for i in range(0, len(master_addresses)):
	final_flag=test_ssh_connection(0,master_addresses[i],host_username)
for i in range(0, len(client_addresses)):
	final_flag = test_ssh_connection(0, client_addresses[i], host_username)
if final_flag ==1:
	sys.exit("ssh测试失败，请检查!")
else:
	print("\033[32m" + "Success: 全部可以ssh免密连接" + "\033[0m")
print("\n")


# 2.配置防火墙和selinux的方法
def set_firwalld_selinux(address,username):
	print("\033[32m" + "{}正在配置防火墙和selinux".format(address + "\033[0m"))
	fir_flag=os.system('ssh {}@{} "systemctl stop firewalld;systemctl disable firewalld"'.format(username,address))
	if fir_flag!=0:
		print("\033[1;31m" + "Failed: 防火墙修改失败" + "\033[0m")
		sys.exit("请检查!")
	sel_flag=os.system("ssh {}@{} 'sed -i 's/SELINUX=.*/SELINUX=disabled/' /etc/selinux/config'".format(username,address))
	if sel_flag!=0:
		print("\033[1;31m" + "Failed: selinux修改失败" + "\033[0m")
		sys.exit("请检查!")
# 配置防火墙和selinux
for i in range(0, len(master_addresses)):
	set_firwalld_selinux(master_addresses[i],host_username)
for i in range(0, len(client_addresses)):
	set_firwalld_selinux(client_addresses[i],host_username)
print("\n")


# 3.配置域名映射
print("\033[32m" + "本地开始配置域名映射" + "\033[0m")
with open("/etc/hosts","w") as f:                      # w重写，a添加，只读
	f.write("127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4\n")
	f.write("::1         localhost localhost.localdomain localhost6 localhost6.localdomain6\n")
	for i in range(0, len(master_addresses)):
		f.write("{} {}\n".format(master_addresses[i],master_domains[i]))
	for i in range(0, len(client_addresses)):
		f.write("{} {}\n".format(client_addresses[i],client_domains[i]))
# 复制本地的/etc/hosts覆盖掉远程主机的/etc/hosts文件
for i in range(0, len(master_addresses)):
	os.system("scp /etc/hosts {}@{}:/etc/hosts".format(host_username,master_addresses[i]))
for i in range(0, len(client_addresses)):
	os.system("scp /etc/hosts {}@{}:/etc/hosts".format(host_username,client_addresses[i]))

# 使用域名首次ssh连接
# 首次域名ssh连接的方法
def first_domain_name_con(domain,username,password):
	print("\033[32m"+"{}首次ssh连接".format(domain)+"\033[0m")
	# BatchMode=yes：表示使SSH在连接过程中不会提示输入密码，而直接尝试免密连接，-o ConnectTimeout=5：表示限制连接超时时间为5秒
	os.system("ssh -o BatchMode=yes -o ConnectTimeout=5 {}@{} 'exit' &> /dev/null".format(username, domain))
	first_domain_flag = os.system("ssh -o BatchMode=yes -o ConnectTimeout=5 {}@{} 'exit'".format(username, domain))
	if first_domain_flag == 0:
		print("\033[32m" + "{}已经可以ssh连接".format(domain) + "\033[0m")
		return
	child = pexpect.spawn('ssh {}@{} "exit"'.format(username,domain))
	try:
		connecting_tuple = child.expect('Are you sure you want to continue connecting')
	except pexpect.exceptions.TIMEOUT:
		print("\033[32m"+"{}已经不是首次ssh连接了".format(domain)+"\033[0m")
	else:
		child.sendline('yes')
	child.expect(pexpect.EOF)               # 用于等待子进程的结束
	print(child.before.decode())            # 等待命令执行完毕并打印输出信息

for i in range(0, len(master_domains)):
	first_domain_name_con(master_domains[i],host_username,host_passwd)
for i in range(0, len(client_domains)):
	first_domain_name_con(client_domains[i], host_username, host_passwd)
	print("\n")

# 4.配置主机名（主机名即域名）
print("\033[32m" + "开始配置主机名" + "\033[0m")
# 配置主机名的方法
def set_hostname(username, address,hostname):
    print("\033[32m" + "{}配置主机名".format(address) + "\033[0m")
    set_hostname_flag=os.system('ssh {}@{} "hostnamectl set-hostname {}"'.format(username, address,hostname))
    if set_hostname_flag != 0:
        print("\033[1;31m" + "Failed: {}配置主机名".format(address) + "\033[0m")
        sys.exit("请检查!")
for i in range(0, len(master_addresses)):
    set_hostname(host_username,master_addresses[i],master_domains[i])
for i in range(0, len(client_addresses)):
    set_hostname(host_username,client_addresses[i],client_domains[i])


# 5.配置yum源
# 配置yum源的方法
def config_local_yum(username,address):
	print("\033[32m" + "{}开始配置本地yum源".format(address) + "\033[0m")
	os.system("ssh {}@{} 'mkdir -p {} && mount {} {}'".format(username,address,yum_mount_dir,yum_mount_dev,yum_mount_dir))
	sed_local_yum='echo "{} {} iso9660 defaults  0  0" >> /etc/fstab'.format(yum_mount_dev,yum_mount_dir)
	os.system("ssh {}@{} '{}'".format(username, address,sed_local_yum))
	os.system("scp /etc/yum.repos.d/centos-local.repo {}@{}:/etc/yum.repos.d/centos-local.repo".format(username,address))
	repolist_flag=os.system("ssh {}@{} '{}'".format(username, address, "yum clean all && yum repolist"))
	if repolist_flag != 0:
		print("\033[1;31m" + "Failed: {}配置yum源失败".format(address) + "\033[0m")
		sys.exit("请检查!")
# 配置扩展源的方法
def config_epel_yum(username,address):
    print("\033[32m" + "{}开始配置扩展源".format(address) + "\033[0m")
    epel_flag=os.system("ssh {}@{} '{}'".format(username, address, "yum install epel-release -y"))
    if epel_flag != 0:
        print("\033[1;31m" + "Failed: {}配置扩展源失败".format(address) + "\033[0m")
        sys.exit("请检查!")
# 配置远程阿里源的方法
def config_remote_yum(username,address):
    print("\033[32m" + "{}开始配置远程阿里源".format(address) + "\033[0m")
    os.system("ssh {}@{} '{}'".format(username, address, "yum install -y wget && wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo"))
    repolist_flag=os.system("ssh {}@{} '{}'".format(username, address, "yum clean all && yum repolist"))
    if repolist_flag != 0:
        print("\033[1;31m" + "Failed: {}配置远程阿里源失败".format(address) + "\033[0m")
        sys.exit("请检查!")

with open("/etc/yum.repos.d/centos-local.repo", "w") as f:  # w重写，a添加，只读
    f.write("[centos7.9]\n")
    f.write("name=centos7.9\n")
    f.write("baseurl=file://{}\n".format(yum_mount_dir))
    f.write("enabled=1\n")
    f.write("gpgcheck=0\n")

for i in range(0, len(master_addresses)):
    config_local_yum(host_username, master_addresses[i])
    config_epel_yum(host_username, master_addresses[i])
    config_remote_yum(host_username, master_addresses[i])
for i in range(0, len(client_addresses)):
    config_local_yum(host_username, client_addresses[i])
    config_epel_yum(host_username, client_addresses[i])
    config_remote_yum(host_username, client_addresses[i])
print("\n")


# 6.安装必要工具
def yum_necessary_tools(username,address):
    print("\033[32m" + "{}开始安装必要工具".format(address) + "\033[0m")
    yum_tool_flag=os.system("ssh {}@{} '{}'".format(username, address,"yum install -y bash-completion vim net-tools tree psmisc lrzsz dos2unix"))
    if yum_tool_flag != 0:
        print("\033[1;31m" + "Failed: {}安装必要工具失败".format(address) + "\033[0m")
        sys.exit("请检查!")
for i in range(0, len(master_addresses)):
    yum_necessary_tools(host_username, master_addresses[i])
for i in range(0, len(client_addresses)):
    yum_necessary_tools(host_username, client_addresses[i])
print("\n")


# 7.配置chrony服务器
print("\033[32m" + "开始配置chrony" + "\033[0m")
# 配置chrony主服务器的方法
def chrony_master_service(username,address):
	print("\033[32m" + "{}配置主chrony".format(address) + "\033[0m")
	# 安装chrony
	chrony_flag = os.system("ssh {}@{} 'yum install -y chrony'".format(username,address))
	if chrony_flag != 0:
		print("\033[1;31m" + "Failed: {}chrony安装失败".format(address) + "\033[0m")
		sys.exit("请检查!")
	# 开启同步地址范围
	chrony_master_allows_addresses = "sed -i 's/#allow 192.168.0.0\/16/allow {}\/24/' /etc/chrony.conf".format(
		chrony_allows_addresses)
	os.system('ssh {}@{} "{}"'.format(username, address, chrony_master_allows_addresses))
	# 开启stratum层数
	chrony_master_allows_stratum = "sed -i 's/#local stratum 10/local stratum 10/' /etc/chrony.conf"
	os.system('ssh {}@{} "{}"'.format(username, address, chrony_master_allows_stratum))
	# 重启服务
	chrony_service = "systemctl restart chronyd && systemctl enable chronyd &> /dev/null"
	os.system('ssh {}@{} "{}"'.format(username, address, chrony_service))
	os.system('ssh {}@{} "sleep 5"'.format(username, address))
	# 开启时间同步
	os.system('ssh {}@{} "timedatectl set-ntp true"'.format(username, address))
# 配置chrony同步节点的方法
def chrony_master_client(username,address):
	print("\033[32m" + "{}配置同步chrony".format(address) + "\033[0m")
	# 安装chrony
	chrony_flag = os.system("ssh {}@{} 'yum install -y chrony'".format(username,address))
	if chrony_flag != 0:
		print("\033[1;31m" + "Failed: {}chrony安装失败".format(address) + "\033[0m")
		sys.exit("请检查!")
	# 删除默认的server地址
	sed_chrony_delete = "sed -i '{}' /etc/chrony.conf".format('/^server/d')
	os.system('ssh {}@{} "{}"'.format(username,address,sed_chrony_delete))
	# 添加自定义的server地址
	for j in range(0, len(master_addresses)):
		sed_chrony_add = "sed -i '{}' /etc/chrony.conf".format("2a\server {} iburst".format(master_addresses[j]))
		os.system('ssh {}@{} "{}"'.format(username, address, sed_chrony_add))
	# 重启服务
	chrony_service = "systemctl restart chronyd && systemctl enable chronyd &> /dev/null"
	os.system('ssh {}@{} "{}"'.format(username,address,chrony_service))
	# 开启时间同步
	os.system('ssh {}@{} "timedatectl set-ntp true"'.format(username, address))
	os.system('ssh {}@{} "sleep 5"'.format(username, address))

	chrony_time = "chronyc sources -v | sed -n '{}'".format("/^\^\*/p")
	chrony_output = subprocess.check_output('ssh {}@{} "{}"'.format(username,address,chrony_time) ,shell=True)
	# 输出结果
	print(chrony_output)
	if chrony_output == "" or chrony_output is None:
		print("\033[1;31m" + "Failed: {}时间同步失败".format(address) + "\033[0m")
		sys.exit("请检查!")

for i in range(0, len(master_addresses)):
	chrony_master_service(host_username,master_addresses[i])
for i in range(0, len(client_addresses)):
	chrony_master_client(host_username,client_addresses[i])
print("\n")


# 8.禁用swap分区，修改linux的内核参数，配置ipvs功能
# 禁用swap分区的方法
def disable_swap(username,address):
    print("\033[32m" + "{}禁用swap分区".format(address) + "\033[0m")
    os.system('ssh {}@{} "{}"'.format(username, address,"sed -i 's/\/dev\/mapper\/centos-swap/#\/dev\/mapper\/centos-swap/' /etc/fstab"))

# 修改linux的内核参数的方法
def update_linux_kernel(username, address):
	print("\033[32m" + "{}修改linux的内核参数".format(address) + "\033[0m")
	os.system("scp /etc/sysctl.d/kubernetes.conf {}@{}:/etc/sysctl.d/kubernetes.conf".format(username,address))
	os.system('ssh {}@{} "sysctl -p"'.format(username, address))
	os.system('ssh {}@{} "modprobe br_netfilter"'.format(username, address))
	os.system('ssh {}@{} "lsmod | grep br_netfilter"'.format(username, address))

# 配置ipvs功能的方法
def config_ipvs(username, address):
	print("\033[32m" + "{}配置ipvs功".format(address) + "\033[0m")
	os.system('ssh {}@{} "yum install -y ipset ipvsadm"'.format(username, address))
	os.system("scp /etc/sysconfig/modules/ipvs.modules {}@{}:/etc/sysconfig/modules/ipvs.modules".format(username,address))
	os.system('ssh {}@{} "chmod +x /etc/sysconfig/modules/ipvs.modules"'.format(username, address))
	os.system('ssh {}@{} "/bin/bash /etc/sysconfig/modules/ipvs.modules"'.format(username, address))
	os.system('ssh {}@{} "lsmod | grep -e ip_vs -e nf_conntrack_ipv4"'.format(username, address))

with open("/etc/sysctl.d/kubernetes.conf", "w") as f:  # w重写，a添加，只读
    f.write("net.bridge.bridge-nf-call-ip6tables = 1\n")
    f.write("net.bridge.bridge-nf-call-iptables = 1\n")
    f.write("net.ipv4.ip_forward = 1")
with open("/etc/sysconfig/modules/ipvs.modules", "w") as f:  # w重写，a添加，只读
    f.write("#!/bin/bash\n")
    f.write("modprobe -- ip_vs\n")
    f.write("modprobe -- ip_vs_rr\n")
    f.write("modprobe -- ip_vs_wrr\n")
    f.write("modprobe -- ip_vs_sh\n")
    f.write("modprobe -- nf_conntrack_ipv4\n")

for i in range(0, len(master_addresses)):
	disable_swap(host_username,master_addresses[i])
	update_linux_kernel(host_username, master_addresses[i])
	config_ipvs(host_username, master_addresses[i])
for i in range(0, len(client_addresses)):
	disable_swap(host_username,client_addresses[i])
	update_linux_kernel(host_username, client_addresses[i])
	config_ipvs(host_username, client_addresses[i])
print("\n")


# 9.重启主机并判断可以ssh登陆（如果要执行此内容需python脚本运行在集群之外的节点，不然手动重启）
def is_ssh_host(username,address):
	print("\033[32m" + "{}开始重启主机".format(address) + "\033[0m")
	os.system('ssh {}@{} "reboot"'.format(username, address))
	os.system('sleep 5')
	for j in range(0,100):
		connect_time_flag=os.system('ssh {}@{} -o ConnectTimeout=5 "exit"'.format(username, address))
		os.system('sleep 3')
		if j == 99:
			print("\033[1;31m" + "Failed: {}设备连接超时".format(address) + "\033[0m\n")
			sys.exit("请检查!")
		if connect_time_flag==0:
			print("\033[32m" + "{}已可以ssh登陆".format(address) + "\033[0m\n")
			break
		else:
			print("\033[32m" + "{}设备正在重启".format(address) + "\033[0m\n")


for i in range(0, len(master_addresses)):
	is_ssh_host(host_username, master_addresses[i])
for i in range(0, len(client_addresses)):
	is_ssh_host(host_username, client_addresses[i])
print("\n")


# 10.安装Docker和k8s组件
# 安装docker的方法
def install_docker(username,address):
	print("\033[32m" + "{}开始安装Docker".format(address) + "\033[0m")
	os.system('ssh {}@{} "wget -O /etc/yum.repos.d/docker-ce.repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo"'.format(username, address))
	os.system('ssh {}@{} "yum install -y --setopt=obsoletes=0 docker-ce-18.06.3.ce-3.el7"'.format(username, address))
	os.system('ssh {}@{} "mkdir -p /etc/docker"'.format(username, address))
	os.system("scp /etc/docker/daemon.json {}@{}:/etc/docker/daemon.json".format(username,address))
	os.system("scp /etc/sysconfig/docker {}@{}:/etc/sysconfig/docker".format(username, address))
	systemctl_docker_flag=os.system('ssh {}@{} "systemctl restart docker && systemctl enable docker"'.format(username, address))
	if systemctl_docker_flag != 0:
		print("\033[1;31m" + "Failed: {}docker服务启动失败，请检查".format(address) + "\033[0m")
		sys.exit("请检查!")
# 安装k8s组件的方法
def install_k8s_module(username,address):
	print("\033[32m" + "{}开始安装k8s组件r".format(address) + "\033[0m")
	os.system("scp /etc/yum.repos.d/kubernetes.repo {}@{}:/etc/yum.repos.d/kubernetes.repo".format(username, address))
	os.system('ssh {}@{} "yum install --setopt=obsoletes=0 kubeadm-1.17.4-0 kubelet-1.17.4-0 kubectl-1.17.4-0 -y"'.format(username, address))
	os.system("scp /etc/sysconfig/kubelet {}@{}:/etc/sysconfig/kubelet".format(username, address))
	systemctl_k8s_flag=os.system('ssh {}@{} "systemctl enable kubelet"'.format(username, address))
	if systemctl_k8s_flag != 0:
		print("\033[1;31m" + "Failed: {}kubelet服务开机自启设置失败，请检查".format(address) + "\033[0m")
		sys.exit("请检查!")

os.system('mkdir -p /etc/docker')
with open("/etc/docker/daemon.json", "w") as f:  # w重写，a添加，只读
	f.write("{\n")
	f.write('"storage-driver": "devicemapper",\n')
	f.write('"exec-opts": ["native.cgroupdriver=systemd"],\n')
	f.write('"registry-mirrors": ["https://ja9e22yz.mirror.aliyuncs.com"]\n')
	f.write("}\n")

with open("/etc/sysconfig/docker", "w") as f:  # w重写，a添加，只读
	f.write("OPTIONS='--selinux-enabled --log-driver=journald --signature-verification=false'\n")

with open("/etc/yum.repos.d/kubernetes.repo", "w") as f:  # w重写，a添加，只读
	f.write("[kubernetes]\n")
	f.write("name=Kubernetes\n")
	f.write("baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64\n")
	f.write("enabled=1\n")
	f.write("gpgcheck=0\n")
	f.write("repo_gpgcheck=0\n")
	f.write("gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg\n")
with open("/etc/sysconfig/kubelet", "w") as f:  # w重写，a添加，只读
	f.write('KUBELET_CGROUP_ARGS="--cgroup-driver=systemd"\n')
	f.write('KUBE_PROXY_MODE="ipvs"\n')

for i in range(0, len(master_addresses)):
	install_docker(host_username, master_addresses[i])
	install_k8s_module(host_username, master_addresses[i])
for i in range(0, len(client_addresses)):
	install_docker(host_username, client_addresses[i])
	install_k8s_module(host_username, client_addresses[i])


# 11.准备集群镜像
def plan_k8s_images(username,address):
	print("\033[32m" + "{}准备集群镜像".format(address) + "\033[0m")
	kubeadm_images_output =subprocess.check_output('ssh {}@{} "kubeadm config images list"'.format(username, address),shell=True)
	kubeadm_images = [line.strip() for line in kubeadm_images_output.split('\n')]
	kubeadm_images.remove("")

	for kubeadm_image in kubeadm_images:
		kubeadm_image=kubeadm_image.split("/")[1]
		print("\033[1;33;40m" + "正在操作{}相关镜像".format(kubeadm_image) + "\033[0m")  # 黄色字体
		os.system('ssh {}@{} "docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/{}"'.format(username, address,kubeadm_image))
		os.system('ssh {}@{} "docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/{} k8s.gcr.io/{}"'.format(username, address,kubeadm_image,kubeadm_image))
		os.system('ssh {}@{} "docker rmi registry.cn-hangzhou.aliyuncs.com/google_containers/{}"'.format(username, address,kubeadm_image))
# 如果后面集群初始化出现镜像缺失问题，请重新拉取镜像
for i in range(0, len(master_addresses)):
	plan_k8s_images(host_username, master_addresses[i])
	plan_k8s_images(host_username, master_addresses[i])
for i in range(0, len(client_addresses)):
	plan_k8s_images(host_username, client_addresses[i])
	plan_k8s_images(host_username, client_addresses[i])
print("\n")



# 12.集群初始化
# 主节点方法
def k8s_init_master(username,address):
	print("\033[32m" + "{}集群主节点初始化".format(address) + "\033[0m")
	kubeadm_init_flag=os.system('ssh {}@{} "kubectl cluster - info &> /dev/null"'.format(username, address))
	if kubeadm_init_flag == 0:
		kubeadm_output=subprocess.check_output('ssh {}@{} "{}"'.format(username, address, "cat $HOME/kubeadm_init.txt"),shell=True)
	else:
		kubeadm_init_command="kubeadm init --kubernetes-version=v1.17.4 --pod-network-cidr=10.244.0.0/16 --service-cidr=10.96.0.0/12 --apiserver-advertise-address={}".format(address)
		kubeadm_output = subprocess.check_output('ssh {}@{} "{}"'.format(username, address,kubeadm_init_command),shell=True)
	lines_with_token = re.findall(r'--token.*', kubeadm_output, re.DOTALL)
	os.system('ssh {}@{} "{}"'.format(username, address, "echo '{}' > $HOME/kubeadm_init.txt".format(kubeadm_output)))
	token_cert_list = lines_with_token[0].split(' ')
	print(token_cert_list)
	kubeadm_codes = []
	for k in range(0,len(token_cert_list)):
		if (token_cert_list[k]=="--token" or token_cert_list[k]=="--discovery-token-ca-cert-hash") and k+1<len(token_cert_list):
			kubeadm_codes.append(token_cert_list[k+1])
	if len(kubeadm_codes)!=2:
		print("\033[1;31m" + "Failed: {}节点初始化失败".format(address) + "\033[0m")
		sys.exit("请检查!")
	os.system('ssh {}@{} "mkdir -p $HOME/.kube"'.format(username, address))
	os.system('ssh {}@{} "cp -i /etc/kubernetes/admin.conf $HOME/.kube/config"'.format(username, address))
	os.system('ssh {}@{} "chown $(id -u):$(id -g) $HOME/.kube/config"'.format(username, address))
	return kubeadm_codes
# 从节点方法
def k8s_init_node(username, address,master_address,kubeadm_codes):
	print("\033[32m" + "{}集群从节点初始化".format(address) + "\033[0m")
	kubeadm_join_command="kubeadm join {}:6443 --token {} \
    --discovery-token-ca-cert-hash {}".format(master_address,kubeadm_codes[0],kubeadm_codes[1])
	kubeadm_join_flag=os.system('ssh {}@{} "{}"'.format(username, address,kubeadm_join_command))
	if kubeadm_join_flag != 0:
		print("\033[1;31m" + "Failed: {}kubeadm join到{}".format(address,master_address) + "\033[0m")
		sys.exit("请检查!")

for i in range(0, len(master_addresses)):
	kubeadm_codes=k8s_init_master(host_username, master_addresses[i])
	for j in range(0, len(client_addresses)):
		k8s_init_node(host_username, client_addresses[j],master_addresses[i],kubeadm_codes)
	os.system('ssh {}@{} "kubectl get nodes"'.format(host_username, master_addresses[i]))
print("\n")


# 12.安装网络插件
def install_kube_flannel(username, address):
	print("\033[32m" + "{}安装网络插件".format(address) + "\033[0m")
	os.system('ssh {}@{} "wget https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml"'.format(username, address))
	os.system('ssh {}@{} "kubectl apply -f kube-flannel.yml"'.format(username, address))
	os.system('sleep 90')
	os.system('ssh {}@{} "kubectl get nodes"'.format(username, address))

for i in range(0, len(master_addresses)):
	install_kube_flannel(host_username, master_addresses[i])
print("\n")