Rockylinux急速安装K8s学习环境

目前在学习k8s，为了更快的了解k8s的各项功能，先使用RockyLinux快速搭建了一个实验环境，使用了1主2从的结构，这里分享一下，k8s小白大佬别喷！

1. 环境初始化(所有主机都要配置)

shell 复制代码

# 网卡配置(不同主机IP要配置不同)
cat /etc/NetworkManager/system-connections/ens18.nmconnection
[connection]
id=ens18
uuid=ee21d8d7-bfeb-34dd-83ca-f72558024a79
type=ethernet
autoconnect-priority=-999
interface-name=ens18
timestamp=1764832445

[ethernet]

[ipv4]
method=manual
address1=192.168.1.190/24,192.168.1.5
dns=114.114.114.114;8.8.8.8

[ipv6]
addr-gen-mode=eui64
method=auto

[proxy]


# 调用 nmcli 重启设备和连接配置
nmcli con up ens18
nmcli con reload

------
# 设置主机名(所有主机都要设置)
root@server190 ~]# cat /etc/hosts

127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.1.190 server190
192.168.1.191 server191
192.168.1.192 server192
------
# Rocky 系统软件源更换
sed -e 's|^mirrorlist=|#mirrorlist=|g' \
    -e 's|^#baseurl=http://dl.rockylinux.org/$contentdir|baseurl=https://mirrors.aliyun.com/rockylinux|g' \
    -i.bak \
    /etc/yum.repos.d/[Rr]ocky*.repo
dnf makecache

# 防火墙修改 firewalld 为 iptables
systemctl stop firewalld
systemctl disable firewalld

yum -y install iptables-services
systemctl start iptables
iptables -F
systemctl enable iptables
service iptables save

# 禁用 Selinux
setenforce 0
sed -i "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config
grubby --update-kernel ALL --args selinux=0

# 设置时区
timedatectl set-timezone Asia/Shanghai
----
# 关闭 swap 分区
swapoff -a
#sed -i 's:/dev/mapper/rlm_192-swap:#/dev/mapper/rlm_192-swap:g' /etc/fstab
sed -ri '/^[^#]*swap/s@^@#@' /etc/fstab
# 修改主机名
hostnamectl set-hostname k8s-node01
---
# 安装 ipvs
yum install -y ipvsadm

# 开启路由转发
echo 'net.ipv4.ip_forward=1' >> /etc/sysctl.conf
sysctl -p'

# 重启系统
reboot

2. 安装Docker(所有主机都要配置)

shell 复制代码

# 加载 bridge
yum install -y epel-release
yum install -y bridge-utils

modprobe br_netfilter
echo 'br_netfilter' >> /etc/modules-load.d/bridge.conf
echo 'net.bridge.bridge-nf-call-iptables=1' >> /etc/sysctl.conf
echo 'net.bridge.bridge-nf-call-iptables=1' >> /etc/sysctl.conf
echo 'net.ipv4.jp_forward=1' >> /etc/sysctl.conf
sysctl -p

# 添加 docker-ce yum 源
# 中科大(ustc)
sudo dnf config-manager --add-repo https://mirrors.ustc.edu.cn/docker-ce/linux/centos/docker-ce.repo
cd /etc/yum.repos.d
# 切换中科大源
sed -e 's|download.docker.com|mirrors.ustc.edu.cn/docker-ce|g' docker-ce.repo >docker-ce-ustc.repo
mv docker-ce.repo docker-ce.repo.back


# 安装 docker-ce
yum -y install docker-ce

# 配置 daemon.
cat > /etc/docker/daemon.json <<EOF
{
    "exec-opts": ["native.cgroupdriver=systemd"],
    "log-driver": "json-file",
    "log-opts": {
    "max-size": "100m",
    "max-file": "100"
    },
    "insecure-registries": ["私服地址"],
    "registry-mirrors": ["可信任的镜像仓库"]
}
EOF
mkdir -p /etc/systemd/system/docker.service.d

# 重启docker服务
systemctl daemon-reload && systemctl restart docker && systemctl enable docker

# 重启系统
reboot

3. 安装Cri-Docker

shell 复制代码

# 安装 cri-docker
wget https://github.com/Mirantis/cri-dockerd/releases/download/v0.3.9/cri-dockerd-0.3.9.amd64.tgz
wget https://github.com/Mirantis/cri-dockerd/releases/download/v0.3.21/cri-dockerd-0.3.21.amd64.tgz
tar -xf cri-dockerd-0.3.9.amd64.tgz
cp cri-dockerd/cri-dockerd /usr/bin/
chmod +x /usr/bin/cri-dockerd

# 配置 cri-docker 服务
cat <<"EOF"> /usr/lib/systemd/system/cri-docker.service
[Unit]
Description=CRI Interface for Docker Application Container Engine
Documentation=https://docs.mirantis.com
After=network-online.target firewalld.service docker.service
Wants=network-online.target
Requires=cri-docker.socket
[Service]
Type=notify
ExecStart=/usr/bin/cri-dockerd --network-plugin=cni --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.9
ExecReload=/bin/kill -s HUP $MAINPID
ExecStop=/bin/kill -s SIGTERM $MAINPID
TimeoutStopSec=30
TimeoutSec=0
RestartSec=2
Restart=always
 
StartLimitBurst=3
 
StartLimitInterval=60s
 
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
 
TasksMax=infinity
Delegate=yes
KillMode=process
 
[Install]
WantedBy=multi-user.target
EOF



---
# 添加 cri-docker 套接字
cat <<"EOF"> /usr/lib/systemd/system/cri-docker.socket
[Unit]
Description=CRI Docker Socket for the API
PartOf=cri-docker.service
 
[Socket]
ListenStream=%t/cri-dockerd.sock
SocketMode=0660
SocketUser=root
SocketGroup=docker
 
[Install]
WantedBy=sockets.target
EOF


# 启动 cri-docker 对应服务
systemctl daemon-reload
systemctl enable cri-docker
systemctl start cri-docker
systemctl is-active cri-docker

4.安装K8s相关软件

shell 复制代码

# 配置k8s阿里云源
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://pkgs.k8s.io/core:/stable:/v1.29/rpm/
enabled=1
gpgcheck=1
gpgkey=https://pkgs.k8s.io/core:/stable:/v1.29/rpm/repodata/repomd.xml.key
# exclude=kubelet kubeadm kubectl cri-tools kubernetes-cni
EOF

yum clean all
yum makecache

# 安装 kubeadm 1.29 版本
yum install -y kubelet-1.29.0 kubectl-1.29.0 kubeadm-1.29.0
systemctl enable kubelet.service

5.初始化集群

shell 复制代码

# 初始化主节点
kubeadm init --apiserver-advertise-address=192.168.1.190 \
--image-repository=registry.aliyuncs.com/google_containers \
--kubernetes-version=1.29.2 \
--service-cidr=10.10.0.0/12 \
--pod-network-cidr=10.244.0.0/16 \
--ignore-preflight-errors=all \
--cri-socket=unix:///var/run/cri-dockerd.sock
# ....
Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

  export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 192.168.1.190:6443 --token 79dfmw.bcdri1p1uc5pbkom \
	--discovery-token-ca-cert-hash sha256:1f30189f89860b28030415566be642f6fc8bb35407819f77da81d282a43895f6
# ....安装完成

# 根创建需要的目录及管理配置文件
mkdir -p $HOME/.kube
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
chown $(id -u):$(id -g) $HOME/.kube/config


# 如果没有注意初始化的token可以通过命令查看
kubeadm token list
# 获取 --discovery-token-ca-cert-hash 值，得到值后需要在前面拼接上 sha256:
openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | \
openssl dgst -sha256 -hex | sed 's/^.* //'


# work token 如果过期后，重新申请
kubeadm token create --print-join-command

shell 复制代码

# 从节点加入主节点，在主节点初始化完成后会出现如何加入集群的命令，可以直接复制,注意要指定容器sock
kubeadm join 192.168.1.190:6443 --token 79dfmw.bcdri1p1uc5pbkom \
	--discovery-token-ca-cert-hash sha256:1f30189f89860b28030415566be642f6fc8bb35407819f77da81d282a43895f6 \
	--cri-socket=unix:///var/run/cri-dockerd.sock

6. 安装网络插件

shell 复制代码

# 官网地址（calico插件）
https://docs.tigera.io/calico/latest/getting-started/kubernetes/self-managed-onprem/onpremises#install-calico-with-kubernetes-api-datastore-more-than-50-nodes

# 安装Calico 命令行工具
curl -o calicoctl -O -L https://github.com/projectcalico/calico/releases/download/v3.26.0/calicoctl-linux-amd64
chmod 755 calicoctl
cp calicoctl  /usr/bin/

# 所有节点都需要进行以下操作(提前下载容器镜像)
wget https://github.com/projectcalico/calico/releases/download/v3.26.0/release-v3.26.0.tgz
tar -xf release-v3.26.0.tgz
cd release-v3.26.0/images
docker load -i calico-cni.tar 
docker load -i calico-node.tar 
docker load -i calico-typha.tar 
docker load -i calico-kube-controllers.tar 

# 主节点操作
curl https://raw.githubusercontent.com/projectcalico/calico/v3.26.3/manifests/calico-typha.yaml -o calico.yaml
vim calico.yaml
# 修改一：
# 取消掉注释
            - name: CALICO_IPV4POOL_CIDR
              value: "10.244.0.0/16"
# value的值改为和 初始化主节点 pod-network-cidr的值相同

# 修改二：
# 修改为 BGP 模式
            # Enable IPIP
            - name: CALICO_IPV4POOL_IPIP
              value: "Off"  # 将Always改成Off
              
# -----------修改三: 可选操作开始---------              
# 目标 IP 或域名可达
- name: calico-node
  image: registry.geoway.com/calico/node:v3.19.1
  env:
    # Auto-detect the BGP IP address.
    - name: IP
      value: "autodetect"
    - name: IP_AUTODETECTION_METHOD
      value: "can-reach=www.baidu.com" # 对机器上所有网卡进行域名解析如果成功就选择 不成功放弃
kubectl set env daemonset/calico-node -n kube-system IP_AUTODETECTION_METHOD=can-reach=www.baidu.com

# 匹配目标网卡
- name: calico-node
  image: registry.geoway.com/calico/node:v3.19.1
  env:
    # Auto-detect the BGP IP address.
    - name: IP
      value: "autodetect"
    - name: IP_AUTODETECTION_METHOD
      value: "interface=eth.*" # 通过正则匹配使用的网卡

# 排除匹配网卡
- name: calico-node
  image: registry.geoway.com/calico/node:v3.19.1
  env:
    # Auto-detect the BGP IP address.
    - name: IP
      value: "autodetect"
    - name: IP_AUTODETECTION_METHOD
      value: "skip-interface=eth.*"
      
# CIDR 指定一个范围来选择网卡
- name: calico-node
  image: registry.geoway.com/calico/node:v3.19.1
  env:
    # Auto-detect the BGP IP address.
    - name: IP
      value: "autodetect"
    - name: IP_AUTODETECTION_METHOD
      value: "cidr=192.168.200.0/24,172.15.0.0/24"

# 修改 kube-proxy 模式为 ipvs
kubectl edit configmap kube-proxy -n kube-system
mode: ipvs
kubectl delete pod -n kube-system -l k8s-app=kube-proxy
# -----------可选操作结束---------   



# 部署 calico             
kubectl  apply -f calico.yaml


# 如果从节点也需要  kubectl get nodes  做以下操作
# 先去到 master节点
scp /etc/kubernetes/admin.conf  192.168.1.191:/etc/kubernetes/
scp /etc/kubernetes/admin.conf  192.168.1.192:/etc/kubernetes/

# 回到node节点
echo "export KUBECONFIG=/etc/kubernetes/admin.conf" >> ~/.bash_profile
source ~/.bash_profile

kubectl get nodes
# 完成

# 可以关注的目录
/etc/kubernetes/mainfests # 这里是启动k8s的清单文件，包括apiserver等，对于想要使用二进制安装的同学有一些参考性 
/etc/kubernetes/admin.conf # k8s的admin管理文件 妥善保管
/var/lib/kubelet/kubeadm-flags.env # kubelet基于kubeadmi的一些子选项，cri接口，镜像仓库