一、前期系统环境准备
准备3台主机：硬盘50G cpu2个 内存2G
1、3台主机同时配置
1）关闭防火墙与selinux、NetworkManager
[root@k8s-master ~]# systemctl stop firewalld
[root@k8s-master ~]# systemctl disable firewalld
Removed symlink /etc/systemd/system/multi-user.target.wants/firewalld.service.
Removed symlink /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service.
[root@k8s-master ~]# setenforce 0
[root@k8s-master ~]# sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/sysconfig/selinux
[root@k8s-master ~]# sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config
[root@k8s-master ~]# systemctl disable --now NetworkManager2）配置yum源
[root@k8s-master yum.repos.d]# ls
CentOS-Base.repo epel.repo
docker-ce.repo epel-testing.repo kubernetes.repo
[root@k8s-master ~]# yum clean all && yum makecache3）配置主机映射
[root@k8s-master ~]# yum -y install vim
[root@k8s-master ~]# vim /etc/hosts
10.0.0.66 k8s-master
10.0.0.77 k8s-node01
10.0.0.88 k8s-node024）配置主机间免密登录
[root@k8s-master ~]# ssh-keygen
[root@k8s-master ~]# ssh-copy-id 10.0.0.77
[root@k8s-master ~]# ssh-copy-id 10.0.0.885）安装必备工具
[root@k8s-master ~]# yum install wget jq psmisc net-tools telnet yum-utils device-mapper-persistent-data lvm2 git tree -y6）关闭swap 分区
[root@k8s-master ~]# swapoff -a && sysctl -w vm.swappiness=0
[root@k8s-master ~]# sed -ri '/^[^#]*swap/s@^@#@' /etc/fstab7）同步时间
[root@k8s-master ~]# yum -y install ntpdate
[root@k8s-master ~]# ntpdate time2.aliyun.com
4 Sep 10:08:59 ntpdate[1897]: adjust time server 203.107.6.88 offset 0.007780 sec
[root@k8s-master ~]# which ntpdate
/usr/sbin/ntpdate
[root@k8s-master ~]# crontab -e
* 5 * * * /usr/sbin/ntpdate time2.aliyun.com
[root@k8s-master ~]# crontab -l
* 5 * * * /usr/sbin/ntpdate time2.aliyun.com8）配置 limit
[root@k8s-master ~]# ulimit -SHn 65535
[root@k8s-master ~]# vim /etc/security/limits.conf
# 末尾添加如下内容
* soft nofile 65536
* hard nofile 131072
* soft nproc 65535
* hard nproc 655350
* soft memlock unlimited
* hard memlock unlimited2、只有master主机配置
1）安装 k8s ⾼可⽤性 Git 仓库并重启
# 在/root/⽬录下克隆⼀个名为k8s-ha-install.git的 Git仓库
[root@k8s-master ~]# cd /root/ ; git clone https://gitee.com/dukuan/k8s-ha-install.git
[root@k8s-master ~]# ls
anaconda-ks.cfg k8s-ha-install
# 后续配置功能性pod的yaml文件
[root@k8s-master k8s-ha-install]# tree -L 2
.
├── calico.yaml
├── krm.yaml
├── LICENSE
├── metrics-server-0.3.7
│ └── components.yaml
├── metrics-server-3.6.1
│ ├── aggregated-metrics-reader.yaml
│ ├── auth-delegator.yaml
│ ├── auth-reader.yaml
│ ├── metrics-apiservice.yaml
│ ├── metrics-server-deployment.yaml
│ ├── metrics-server-service.yaml
│ └── resource-reader.yaml
└── README.md
2 directories, 12 files二、配置内核模块
1、3台主机同时配置
使用该工具可以同时操作多个主机
1）配置ipvs模块
[root@k8s-master ~]# yum install ipvsadm ipset sysstat conntrack libseccomp -y
[root@k8s-master ~]# modprobe -- ip_vs
[root@k8s-master ~]# modprobe -- ip_vs_rr
[root@k8s-master ~]# modprobe -- ip_vs_wrr
[root@k8s-master ~]# modprobe -- ip_vs_sh
[root@k8s-master ~]# modprobe -- nf_conntrack
# 在系统启动时加载下列 IPVS 和相关功能所需的模块
[root@k8s-master ~]# vim /etc/modules-load.d/ipvs.config
ip_vs
ip_vs_lc
ip_vs_wlc
ip_vs_rr
ip_vs_wrr
ip_vs_lblc
ip_vs_lblcr
ip_vs_dh
ip_vs_sh
ip_vs_fo
ip_vs_nq
ip_vs_sed
ip_vs_ftp
ip_vs_sh
nf_conntrack
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip
# 加载系统内核参数并应用它们
[root@k8s-master ~]# sysctl --system
# 开机⾃启systemd默认提供的⽹络管理服务
[root@k8s-master ~]# systemctl enable systemd-modules-load.service
[root@k8s-master ~]# systemctl start systemd-modules-load.service
# 在已加载的内核模块列表中查找与 ip_vs（IP Virtual Server，IP 虚拟服务器）和 nf_conntrack（Netfilter Connection Tracking，网络过滤器连接跟踪）相关的模块信息
[root@k8s-master ~]# lsmod | grep -e ip_vs -e nf_conntrack
ip_vs_sh 12688 0
ip_vs_wrr 12697 0
ip_vs 141432 4 ip_vs_sh,ip_vs_wrr
nf_conntrack 133053 1 ip_vs
libcrc32c 12644 3 xfs,ip_vs,nf_conntrack2）配置k8s内核
[root@k8s-master ~]# vim /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
fs.may_detach_mounts = 1
net.ipv4.conf.all.route_localnet = 1
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
fs.file-max=52706963
fs.nr_open=52706963
net.netfilter.nf_conntrack_max=2310720
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.tcp_keepalive_intvl =15
net.ipv4.tcp_max_tw_buckets = 36000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_max_orphans = 327680
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.ip_conntrack_max = 65536
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.tcp_timestamps = 0
# 保存后，所有节点重启，保证重启后内核依然加载
[root@k8s-master ~]# reboot三、基本组件安装
1、3台主机同时配置
1）安装 Containerd
# 卸载之前的containerd
[root@k8s-master ~]# yum remove -y podman runc containerd
# 安装Docker和containerd
[root@k8s-master ~]# yum install containerd.io docker-ce dockerce-cli -y
[root@k8s-master ~]# yum list installed | grep docker
containerd.io.x86_64 1.6.33-3.1.el7 @docker-ce-stable
docker-buildx-plugin.x86_64 0.14.1-1.el7 @docker-ce-stable
docker-ce.x86_64 3:26.1.4-1.el7 @docker-ce-stable
docker-ce-cli.x86_64 1:26.1.4-1.el7 @docker-ce-stable
docker-ce-rootless-extras.x86_64
26.1.4-1.el7 @docker-ce-stable
docker-compose-plugin.x86_64 2.27.1-1.el7 @docker-ce-stable2）配置 Containerd 所需模块
[root@k8s-master ~]# cat <<EOF | sudo tee /etc/modules-load.d/containerd.conf
overlay
br_netfilter
EOF
[root@k8s-master ~]# modprobe -- overlay
[root@k8s-master ~]# modprobe -- br_netfilter3）配置 Containerd 所需内核
[root@k8s-master ~]# cat <<EOF | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF
[root@k8s-master ~]# sysctl --system4）Containerd 配置⽂件
[root@k8s-master ~]# mkdir -p /etc/containerd
# 读取containerd的配置并保存到/etc/containerd/config.toml
[root@k8s-master ~]# containerd config default | tee /etc/containerd/config.toml
[root@k8s-master ~]# vim /etc/containerd/config.toml
# 找到第63行修改为sandbox_image = "registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.9"
# 找到containerd.runtimes.runc.options模块，添加SystemdCgroup = true，如果已经存在则直接修改（在第127行）
# 添加sandbox_image = "registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.9"（第128行）
# 加载systemctl控制脚本
[root@k8s-master ~]# systemctl daemon-reload
# 启动containerd并设置开机启动
[root@k8s-master ~]# systemctl enable --now containerd
Created symlink from /etc/systemd/system/multi-user.target.wants/containerd.service to /usr/lib/systemd/system/containerd.service.5）配置 crictl 客户端连接的运⾏位置
# 配置容器运⾏环境的crictl.yml⽂件
[root@k8s-master ~]# cat <<EOF | sudo tee /etc/crictl.yaml
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
timeout: 10
debug: false
EOF6）安装 Kubernetes 组件
# 安装 Kubeadm、Kubelet 和 Kubectl
[root@k8s-master ~]# yum install kubeadm-1.28* kubelet-1.28* kubectl-1.28* -y
[root@k8s-master ~]# systemctl daemon-reload
[root@k8s-master ~]# systemctl enable --now kubelet
Created symlink from /etc/systemd/system/multi-user.target.wants/kubelet.service to /usr/lib/systemd/system/kubelet.service.
[root@k8s-master ~]# yum list installed | grep kube
cri-tools.x86_64 1.26.0-0 @kubernetes
kubeadm.x86_64 1.28.2-0 @kubernetes
kubectl.x86_64 1.28.2-0 @kubernetes
kubelet.x86_64 1.28.2-0 @kubernetes
kubernetes-cni.x86_64 1.2.0-0 @kubernetes问题解决：kubelet启动失败
# 查看日志
[root@k8s-master ~]# vim /var/log/messages
# 配置文件未生成，重新安装kubelet
# 问题解决：
[root@k8s-master ~]# yum -y remove kubelet
[root@k8s-master ~]# yum -y install kubelet-1.28*
[root@k8s-master ~]# systemctl start kubelet
[root@k8s-master ~]# systemctl status kubelet
Active: active (running) since 三 2024-09-11 14:25:57 CST; 3s ago
# 由于kubeadm依赖kubelet所以卸载前者时后者也卸载了，需要重新安装
[root@k8s-master ~]# yum -y install kubeadm-1.28*
# 查看kubelet端口是否启动
[root@k8s-master ~]# netstat -lntup | grep kube
tcp 0 0 127.0.0.1:10248 0.0.0.0:* LISTEN 2392/kubelet
tcp6 0 0 :::10250 :::* LISTEN 2392/kubelet
tcp6 0 0 :::10255 :::* LISTEN 2392/kubelet 2、只有master主机配置（Kubernetes 集群初始化）
1）Kubeadm 配置⽂件
[root@k8s-master ~]# vim kubeadm-config.yaml
# 粘贴文件内容并修改文件
# 修改第12行、24行、29行的ip地址为自己本机的ip地址
piVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
token: 7t2weq.bjbawausm0jaxury
ttl: 24h0m0s
usages:
- signing
- authentication
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 10.0.0.66
bindPort: 6443
nodeRegistration:
criSocket: unix:///var/run/containerd/containerd.sock
name: k8s-master
taints:
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
---
apiServer:
certSANs:
- 10.0.0.66
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: 10.0.0.66:6443
controllerManager: {}
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: v1.28.2
networking:
dnsDomain: cluster.local
podSubnet: 172.16.0.0/16
serviceSubnet: 10.96.0.0/16
scheduler: {}
# 将旧的kubeadm配置⽂件转换为新的格式
[root@k8s-master ~]# kubeadm config migrate --old-config kubeadm-config.yaml --new-config new.yaml2）下载组件镜像
# 通过新的配置⽂件new.yaml从指定的阿⾥云仓库拉取kubernetes组件镜像
[root@k8s-master ~]# kubeadm config images pull --config /root/new.yaml 3）集群初始化
[root@k8s-master ~]# kubeadm init --config /root/new.yaml --upload-certs
# 根据提示信息完成配置
[root@k8s-master ~]# mkdir -p $HOME/.kube
[root@k8s-master ~]# sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
[root@k8s-master ~]# sudo chown $(id -u):$(id -g) $HOME/.kube/config
# 将node结点加入集群的信息保存到一个文件中，以便使用
[root@k8s-master ~]# vim k8s.txt
kubeadm join 10.0.0.66:6443 --token 7t2weq.bjbawausm0jaxury \
--discovery-token-ca-cert-hash sha256:f3ac431e03dae7f972728eb71eef1828264d42ec20a163893c812a2a0289cf99问题解决：集群初始化失败
# 端口18258正被kubelet使用，初始化会自动启动kubelet，所以手动关闭kubelet服务
[root@k8s-master ~]# systemctl stop kubelet
# 修改ip_forward文件内容
[root@k8s-master ~]# echo 1 > /proc/sys/net/ipv4/ip_forward
[root@k8s-master ~]# kubeadm init --config /root/new.yaml --upload-certs
# 错误信息显示本机内存不够，cpu数量不够，我们现在将本机内存提到4个G，cpu数量提到4个
# 注意要关闭本主机然后进行修改主机配置的操作
[root@k8s-master ~]# echo 1 > /proc/sys/net/ipv4/ip_forward
[root@k8s-master ~]# kubeadm init --config /root/new.yaml --upload-certs
# 检查kubelet为运行状态
[root@master ~]# systemctl status kubelet
Active: active (running) since 五 2024-09-06 17:33:30 CST; 5min ago
# 可能是配置文件的地址没有改，所以找不到主机，所以超时
[root@k8s-master ~]# vim new.yaml
# 修改第12行、24行、29行的ip地址为自己本机的ip地址
# 初始化重置
[root@k8s-master ~]# kubeadm reset -f ; ipvsadm --clear ; rm -rf ~/.kube
[root@k8s-master ~]# kubeadm init --config /root/new.yaml --upload-certs4）加载环境变量
[root@k8s-master ~]# vim /root/.bashrc
export KUBECONFIG=/etc/kubernetes/admin.conf
[root@k8s-master ~]# source /root/.bashrc5）查看组件容器状态
| 状态名称 | 中文 | 说明 |
|---|---|---|
| pending | 挂起 | 当前pod没有工作 |
| running | 运行中 | 当前pod正常工作 |
| containercreating | 正在创建容器 | 正在创建容器 |
[root@k8s-master ~]# kubectl get po -A
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system coredns-6554b8b87f-2v4tx 0/1 Pending 0 52m
kube-system coredns-6554b8b87f-zfqlb 0/1 Pending 0 52m
kube-system etcd-k8s-master 1/1 Running 0 52m
kube-system kube-apiserver-k8s-master 1/1 Running 0 52m
kube-system kube-controller-manager-k8s-master 1/1 Running 0 52m
kube-system kube-proxy-9r6st 1/1 Running 0 52m
kube-system kube-proxy-lx5wz 1/1 Running 0 22m
kube-system kube-proxy-xmk6s 1/1 Running 0 25m
kube-system kube-scheduler-k8s-master 1/1 Running 0 52m6）查看集群信息
[root@k8s-master ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
k8s-master NotReady control-plane 25s v1.28.27）Token 过期处理
Token 过期后⽣成新的 token：
kubeadm token create --print-join-commandMaster 需要⽣成 --certificate-key：
kubeadm init phase upload-certs --upload-certs3、node结点执行
1）加入集群
[root@k8s-node01 ~]# kubeadm join 10.0.0.66:6443 --token 7t2weq.bjbawausm0jaxury \
> --discovery-token-ca-cert-hash sha256:f3ac431e03dae7f972728eb71eef1828264d42ec20a163893c812a2a0289cf99问题解决：加入集群失败
# 端口被占用，手动停止kubelet，加入集群的过程中会自动启动
[root@k8s-node01 ~]# systemctl stop kubelet
Warning: kubelet.service changed on disk. Run 'systemctl daemon-reload' to reload units.
# 修改ip_forward文件
[root@k8s-node01 ~]# echo 1 > /proc/sys/net/ipv4/ip_forward
[root@k8s-node01 ~]# kubeadm join 10.0.0.66:6443 --token 7t2weq.bjbawausm0jaxury --discovery-token-ca-cert-hash sha256:f3ac431e03dae7f972728eb71eef1828264d42ec20a163893c812a2a0289cf994、master主机执行（Calico 组件安装）
1）查看集群状态与容器状态
[root@k8s-master ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
k8s-master NotReady control-plane 31m v1.28.2
k8s-node01 NotReady <none> 4m4s v1.28.2
k8s-node02 NotReady <none> 57s v1.28.2
[root@k8s-master ~]# kubectl get po -A
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system coredns-6554b8b87f-2v4tx 0/1 Pending 0 52m
kube-system coredns-6554b8b87f-zfqlb 0/1 Pending 0 52m
kube-system etcd-k8s-master 1/1 Running 0 52m
kube-system kube-apiserver-k8s-master 1/1 Running 0 52m
kube-system kube-controller-manager-k8s-master 1/1 Running 0 52m
kube-system kube-proxy-9r6st 1/1 Running 0 52m
kube-system kube-proxy-lx5wz 1/1 Running 0 22m
kube-system kube-proxy-xmk6s 1/1 Running 0 25m
kube-system kube-scheduler-k8s-master 1/1 Running 0 52m
[root@k8s-master ~]# kubectl get po -Aowide
NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
kube-system coredns-6554b8b87f-2v4tx 0/1 Pending 0 53m <none> <none> <none> <none>
kube-system coredns-6554b8b87f-zfqlb 0/1 Pending 0 53m <none> <none> <none> <none>
kube-system etcd-k8s-master 1/1 Running 0 54m 10.0.0.66 k8s-master <none> <none>
kube-system kube-apiserver-k8s-master 1/1 Running 0 54m 10.0.0.66 k8s-master <none> <none>
kube-system kube-controller-manager-k8s-master 1/1 Running 0 54m 10.0.0.66 k8s-master <none> <none>
kube-system kube-proxy-9r6st 1/1 Running 0 53m 10.0.0.66 k8s-master <none> <none>
kube-system kube-proxy-lx5wz 1/1 Running 0 23m 10.0.0.88 k8s-node02 <none> <none>
kube-system kube-proxy-xmk6s 1/1 Running 0 26m 10.0.0.77 k8s-node01 <none> <none>
kube-system kube-scheduler-k8s-master 1/1 Running 0 54m 10.0.0.66 k8s-master <none> <none>2）部署calico的pod
# 找到配置文件calico
[root@k8s-master ~]# cd k8s-ha-install/
# 切换 git 分⽀
[root@k8s-master k8s-ha-install]# git checkout manual-installation-v1.28.x
分支 manual-installation-v1.28.x 设置为跟踪来自 origin 的远程分支 manual-installation-v1.28.x。
切换到一个新分支 'manual-installation-v1.28.x'
# 修改 Pod ⽹段
[root@k8s-master k8s-ha-install]# ls
bootstrap CoreDNS dashboard metrics-server README.md
calico csi-hostpath kubeadm-metrics-server pki snapshotter
[root@k8s-master k8s-ha-install]# cd calico/
[root@k8s-master calico]# ls
calico.yaml
[root@k8s-master calico]# vim /etc/kubernetes/manifests/kube-controller-manager.yaml
# 获取已定义的Pod⽹段
[root@k8s-master calico]# POD_SUBNET=`cat /etc/kubernetes/manifests/kube-controller-manager.yaml | grep cluster-cidr= | awk -F= '{print $NF}'`
[root@k8s-master calico]# echo $POD_SUBNET
172.16.0.0/16
# 修改配置文件，将文件中的POD_CIDR替换成172.16.0.0/16
[root@k8s-master calico]# sed -i "s#POD_CIDR#${POD_SUBNET}#g" calico.yaml
# 创建pod
[root@k8s-master calico]# kubectl apply -f calico.yaml 3）查看容器状态
[root@k8s-master calico]# kubectl get po -A
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system calico-kube-controllers-6d48795585-v5d7x 0/1 Pending 0 69s
kube-system calico-node-747k8 0/1 Init:0/3 0 69s
kube-system calico-node-7klq9 0/1 Init:0/3 0 69s
kube-system calico-node-j9b44 0/1 Init:0/3 0 69s
kube-system coredns-6554b8b87f-2v4tx 0/1 Pending 0 104m
kube-system coredns-6554b8b87f-zfqlb 0/1 Pending 0 104m
kube-system etcd-k8s-master 1/1 Running 0 104m
kube-system kube-apiserver-k8s-master 1/1 Running 0 104m
kube-system kube-controller-manager-k8s-master 1/1 Running 1 (7m42s ago) 7m27s
kube-system kube-proxy-9r6st 1/1 Running 0 104m
kube-system kube-proxy-lx5wz 1/1 Running 0 74m
kube-system kube-proxy-xmk6s 1/1 Running 0 77m
kube-system kube-scheduler-k8s-master 1/1 Running 0 104m