K8s-1.29.2二进制安装-第一章

从本章来完成安装k8s学习的最后一种安装方式(二进制安装)，系统使用Rockly9.6，K8s版本1.29.2，一共会分成几张进行编写。

1. 安装Topo

2.环境初始化

1、镜像下载(所有节点)

官方下载地址

https://rockylinux.org/download

阿里云镜像下载地址

https://mirrors.aliyun.com/rockylinux/9/isos/x86_64/?spm=a2c6h.25603864.0.0.29696621VzJej5

2、环境配置(所有节点)

2.1 网卡配置(所有主机都要设置好IP)

# cat /etc/NetworkManager/system-connections/ens160.nmconnection
[ipv4]
method=manual
address1=192.168.1.12/24,192.168.1.1
dns=114.114.114.114;8.8.8.8

2.2 修改主机名和/etc/hosts解析文件

# 设置主机名
hostnamectl set-hostname server170
# Host文件解析
# cat /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.1.170 server170
192.168.1.171 server171
192.168.1.172 server172
192.168.1.173 server173
192.168.1.174 server174

2.3 Rocky 系统软件源更换

# 使用阿里源
sed -e 's|^mirrorlist=|#mirrorlist=|g' \
  -e 's|^#baseurl=http://dl.rockylinux.org/$contentdir|baseurl=https://mirrors.aliyun.com/rockylinux|g' \
  -i.bak \
  /etc/yum.repos.d/[Rr]ocky*.repo
# 使用私有仓库
sed -e 's|^mirrorlist=|#mirrorlist=|g' \
  -e 's|^#baseurl=http://dl.rockylinux.org/$contentdir|baseurl=http://192.168.1.12:8081/repository/Rockylinux-YumGroup|g' \
  -i.bak \
  /etc/yum.repos.d/[Rr]ocky*.repo

dnf makecache

2.4 防火墙修改 firewalld 为 iptables

systemctl stop firewalld
systemctl disable firewalld
yum -y install iptables-services
systemctl start iptables
iptables -F # 清理iptables规则
systemctl enable iptables

2.5 禁用 Selinux

setenforce 0
sed -i "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config
grubby --update-kernel ALL --args selinux=0

2.6 设置时区及时间同步

# 将3台master构建成高可用时间服务器，2台node作为客户端向时间服务器同步
# 同步时间
timedatectl set-timezone Asia/Shanghai

# 3台服务端(master)
yum install chrony -y

cat > /etc/chrony.conf << EOF 
pool ntp1.aliyun.com iburst
pool ntp2.aliyun.com iburst
pool ntp3.aliyun.com iburst
driftfile /var/lib/chrony/drift
makestep 1.0 3
rtcsync
allow 192.168.1.0/24
local stratum 10 # 设置权重，这个3台服务器要不同
keyfile /etc/chrony.keys
leapsectz right/UTC
logdir /var/log/chrony
EOF

systemctl restart chronyd ; systemctl enable chronyd

# 客户端(node)
yum install chrony -y

cat > /etc/chrony.conf << EOF 
pool 192.168.1.170 iburst
pool 192.168.1.171 iburst
pool 192.168.1.172 iburst
driftfile /var/lib/chrony/drift
makestep 1.0 3
rtcsync
keyfile /etc/chrony.keys
leapsectz right/UTC
logdir /var/log/chrony
EOF

systemctl restart chronyd ; systemctl enable chronyd

# 使用客户端进行验证
chronyc sources -v

2.7 关闭 swap 分区

swapoff -a
sed -i 's:/dev/mapper/rl-swap:#/dev/mapper/rl-swap:g' /etc/fstab

2.8 安装配置IPVs

# 安装 ipvs
yum install ipvsadm ipset sysstat conntrack libseccomp -y

cat >> /etc/modules-load.d/ipvs.conf <<EOF 
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip
EOF

systemctl restart systemd-modules-load.service

lsmod | grep -e ip_vs -e nf_conntrack

2.9 安装必备的工具包

yum update -y && yum -y install wget psmisc vim net-tools nfs-utils telnet yum-utils device-mapper-persistent-data lvm2 git tar curl 

2.10 将calico网卡排除被NetworkManager所管理(后期会用calico进行管理)

cat > /etc/NetworkManager/conf.d/calico.conf << EOF 
[keyfile]
unmanaged-devices=interface-name:cali*;interface-name:tunl*
EOF

systemctl restart NetworkManager

2.11 内核参数优化

# 配置 ulimit
ulimit -SHn 65535

cat >> /etc/security/limits.conf <<EOF
* soft nofile 655360
* hard nofile 131072
* soft nproc 655350
* hard nproc 655350
* seft memlock unlimited
* hard memlock unlimitedd
EOF

# 修改内核参数
cat <<EOF > /etc/sysctl.d/k8s.conf
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
fs.may_detach_mounts = 1
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
fs.file-max=52706963
fs.nr_open=52706963
net.netfilter.nf_conntrack_max=2310720
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.tcp_keepalive_intvl =15
net.ipv4.tcp_max_tw_buckets = 36000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_max_orphans = 327680
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.ip_conntrack_max = 65536
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.tcp_timestamps = 0
net.core.somaxconn = 16384
net.ipv6.conf.all.disable_ipv6 = 0
net.ipv6.conf.default.disable_ipv6 = 0
net.ipv6.conf.lo.disable_ipv6 = 0
net.ipv6.conf.all.forwarding = 1
EOF

sysctl --system

3、 安装 Docker

# 二进制包下载地址：https://download.docker.com/linux/static/stable/x86_64/
# wget https://mirrors.ustc.edu.cn/docker-ce/linux/static/stable/x86_64/docker-25.0.3.tgz

tar xf docker-*.tgz 
cp docker/* /usr/bin/

# 创建 containerd 的 service 文件,并且启动
cat >/etc/systemd/system/containerd.service <<EOF
[Unit]
Description=containerd container runtime
Documentation=https://containerd.io
After=network.target local-fs.target

[Service]
ExecStartPre=-/sbin/modprobe overlay
ExecStart=/usr/bin/containerd
Type=notify
Delegate=yes
KillMode=process
Restart=always
RestartSec=5
LimitNPROC=infinity
LimitCORE=infinity
LimitNOFILE=1048576
TasksMax=infinity
OOMScoreAdjust=-999

[Install]
WantedBy=multi-user.target
EOF

systemctl enable --now containerd.service

# 准备 docker 的 service 文件
cat > /etc/systemd/system/docker.service <<EOF
[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network-online.target firewalld.service cri-docker.service docker.socket containerd.service
Wants=network-online.target
Requires=docker.socket containerd.service

[Service]
Type=notify
ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock
ExecReload=/bin/kill -s HUP $MAINPID
TimeoutSec=0
RestartSec=2
Restart=always
StartLimitBurst=3
StartLimitInterval=60s
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
TasksMax=infinity
Delegate=yes
KillMode=process
OOMScoreAdjust=-500

[Install]
WantedBy=multi-user.target
EOF

#准备 docker 的 socket 文件
cat > /etc/systemd/system/docker.socket <<EOF
[Unit]
Description=Docker Socket for the API

[Socket]
ListenStream=/var/run/docker.sock
SocketMode=0660
SocketUser=root
SocketGroup=docker

[Install]
WantedBy=sockets.target
EOF

# 配置加速器
mkdir /etc/docker/ -pv

cat >/etc/docker/daemon.json <<EOF
{
 "exec-opts": ["native.cgroupdriver=systemd"],
 "registry-mirrors": [
  "https://docker.m.daocloud.io"
 ],
 "max-concurrent-downloads": 10,
 "log-driver": "json-file",
 "log-level": "warn",
 "log-opts": {
  "max-size": "10m",
  "max-file": "3"
  },
 "data-root": "/var/lib/docker"
}
EOF

# 启动 Docker
groupadd docker
systemctl daemon-reload
systemctl enable --now docker.socket
systemctl enable --now docker.service
systemctl status docker.service
docker info

4、安装 cri-docker

# 解压 cri-docker
# https://github.com/Mirantis/cri-dockerd/releases/
# wget https://github.com/Mirantis/cri-dockerd/releases/download/v0.3.10/cri-dockerd-0.3.10.amd64.tgz

tar xvf cri-dockerd-*.amd64.tgz 
cp cri-dockerd/cri-dockerd /usr/bin/
chmod +x /usr/bin/cri-dockerd
# 写入启动  cri-docker 配置文件

cat >  /usr/lib/systemd/system/cri-docker.service <<EOF
[Unit]
Description=CRI Interface for Docker Application Container Engine
Documentation=https://docs.mirantis.com
After=network-online.target firewalld.service docker.service
Wants=network-online.target
Requires=cri-docker.socket

[Service]
Type=notify
ExecStart=/usr/bin/cri-dockerd --network-plugin=cni --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.7
ExecReload=/bin/kill -s HUP $MAINPID
TimeoutSec=0
RestartSec=2
Restart=always
StartLimitBurst=3
StartLimitInterval=60s
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
TasksMax=infinity
Delegate=yes
KillMode=process

[Install]
WantedBy=multi-user.target
EOF

# 写入 cri-docker 的 socket 配置文件

cat > /usr/lib/systemd/system/cri-docker.socket <<EOF
[Unit]
Description=CRI Docker Socket for the API
PartOf=cri-docker.service

[Socket]
ListenStream=%t/cri-dockerd.sock
SocketMode=0660
SocketUser=root
SocketGroup=docker

[Install]
WantedBy=sockets.target
EOF

# 启动 cri-docker
systemctl daemon-reload
systemctl enable --now cri-docker.service
systemctl status cri-docker