k8s安装部署（利用ansible）

|--------|---------------|
| 主机 | ip |
| harbor | 172.25.254.11 |
| master | 172.25.254.12 |
| node1 | 172.25.254.13 |
| node2 | 172.25.254.14 |

在/etc/hosts解析：

172.25.254.11 harbor reg.timinglee.org

172.25.254.12 master

172.25.254.13 node1

172.25.254.14 node2

1.部署docker本地仓库

$root@harbor \~$ # cat > /etc/yum.repos.d/docker.repo <<EOF
$docker$
name = docker

baseurl = https://mirrors.aliyun.com/docker-ce/linux/rhel/9.6/x86_64/stable/

gpgcheck = 0

EOF

$root@harbor \~$ # cat > /etc/yum.repos.d/docker.repo <<EOF
$docker$
name = docker

baseurl = https://mirrors.aliyun.com/docker-ce/linux/rhel/9.6/x86_64/stable/

gpgcheck = 0

EOF

$root@harbor \~$ # dnf install httpd createrepo-y

$root@harbor \~$ # mkdir /var/www/html/docker/ -p

$root@harbor \~$ # vim /etc/httpd/conf/httpd.conf

Listen 4444

$root@harbor \~$ # systemctl enable --now httpd

$root@harbor \~$ # dnf install docker-ce -downloadonly --destdir /mnt/ -y

$root@harbor \~$ # mv /mnt/*.rpm /var/www/html/docker/

$root@harbor \~$ # createrepo -v /var/www/html/docker/

$root@harbor \~$ # cat > /etc/yum.repos.d/docker.repo <<EOF
$docker$
name = docker

baseurl = http://172.25.254.11:4444/docker

gpgcheck = 0

EOF

$root@harbor \~$ # dnf install docker-ce -y

2.安装docker在harbor仓库节点

$root@harbor \~$ # dnf install docker-ce -y

$root@harbor \~$ # echo br_netfilter > /etc/modules-load.d/docker_mod.conf

$root@harbor \~$ # modprobe -a br_netfilter

$root@harbor \~$ # vim /etc/sysctl.d/docker.conf

net.bridge.bridge-nf-call-iptables = 1

net.bridge.bridge-nf-call-ip6tables = 1

net.ipv4.ip_forward = 1

$root@harbor \~$ # systemctl restart systemd-modules-load.service

$root@harbor \~$ # sysctl --system

$root@harbor \~$ # vim /lib/systemd/system/docker.service

ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --iptables=true

$root@harbor \~$ # systemctl daemon-reload

$root@harbor \~$ # systemctl enable --now docker

2.生成key

$root@harbor \~$ # mkdir /data/certs -p

$root@harbor \~$ # mkdir /data/certs -p

$root@harbor \~$ # openssl req -newkey rsa:4096 \

-nodes -sha256 -keyout /data/certs/timinglee.org.key \

-addext "subjectAltName = DNS:reg.timinglee.org" \

-x509 -days 365 -out /data/certs/timinglee.org.crt

You are about to be asked to enter information that will be incorporated

into your certificate request.

What you are about to enter is what is called a Distinguished Name or a DN.

There are quite a few fields but you can leave some blank

For some fields there will be a default value,

If you enter '.', the field will be left blank.

Country Name (2 letter code) $XX$ :CN

State or Province Name (full name) \[\]:Shannxi

Locality Name (eg, city) $Default City$ :Xi'an

Organization Name (eg, company) $Default Company Ltd$ :kubernetes

Organizational Unit Name (eg, section) \[\]:harbor

Common Name (eg, your name or your server's hostname) \[\]:reg.timinglee.org

Email Address \[\]:admin@timinglee.org

3.编辑harbor配置文件

$root@harbor \~$ # tar zxf harbor-offline-installer-v2.5.4.tgz -C /opt/

$root@harbor \~$ # cd /opt/harbor/

$root@harbor harbor$ # ls

common.sh harbor.v2.5.4.tar.gz harbor.yml.tmpl install.sh LICENSE prepare

$root@harbor harbor$ # cp harbor.yml.tmpl harbor.yml

$root@harbor harbor$ # vim harbor.yml

一.安装ansible

$root@harbor \~$ # vim /etc/yum.repos.d/epel.repo
$epel$
name = epel

baseurl = https://mirrors.aliyun.com/epel/9/Everything/x86_64/

gpgcheck = 0

$root@harbor \~$ # dnf install ansible-core.x86_64 ansible.noarch -y

二.配置所需用户

三.生成ansible的临时配置文件并建立运行环境

vim inventory

vim ansible.cfg

做免密登陆：

#先切devops

su - devops

#生成devops自己的密钥

ssh-keygen -t ed25519

#全部回车，不要输入密码

#推送公钥到被控机root账号

ssh-copy-id root@172.25.254.12

ssh-copy-id root@172.25.254.13

ssh-copy-id root@172.25.254.14

$devops@harbor ansible$ $ ansible all -m ping

在所有被控服务器上：

创建普通用户 devops

设置密码：lee

赋予 devops 免密 sudo 权限，可以 sudo 变成 root，不用输密码

$devops@harbor ansible$ $ ansible all -m user -a "name=devops state=present"

$devops@harbor ansible$ $ ansible all -m shell -a 'echo "lee" | passwd --stdin devops'

$devops@harbor ansible$ $ ansible all -m lineinfile -a 'path=/etc/sudoers line="devops ALL=(ALL) NOPASSWD: ALL" insertafter=EOF'

四.生成ansible的主配置文件并测试

五.利用ansible剧本部署docker

vim kubernetes-devops.yml

bash 复制代码

- name: devops k8s from ansible
  hosts: all
  tasks:
  - name: setup docker repo
    yum_repository:
      name: docker
      description: docker
      baseurl: http://172.25.254.11:4444/docker
      file: docker
      gpgcheck: no

  - name: install docker
    dnf:
      name: docker-ce
      state: present

  - name: setup docker.service
    replace:
      path: /lib/systemd/system/docker.service
      regexp: "ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock"
      replace: "ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock  --iptables=true"

  - name: setup docker registry
    copy:
      content: |
        {
          "registry-mirrors":["https://reg.timinglee.org"]
        }
      dest: /etc/docker/daemon.json

  - name: setup docker certs
    file:
      path: /etc/docker/certs.d/reg.timinglee.org
      state: directory

  - name: cp certs file
    copy:
      src: "{{item.src}}"
      dest: "{{item.dest}}"
    loop:
      - src: /etc/docker/certs.d/reg.timinglee.org/ca.crt
        dest: /etc/docker/certs.d/reg.timinglee.org/ca.crt
      - src: /etc/docker/certs.d/reg.timinglee.org/ca.crt
        dest: /etc/pki/ca-trust/source/anchors/ca.crt

  - name: load module
    copy:
      dest: "{{item.dest}}"
      content: "{{item.value}}"
    loop:
      - dest: /etc/modules-load.d/docker_mod.conf
        value: br_netfilter
      - dest: /etc/sysctl.d/docker.conf
        value: |
          net.bridge.bridge-nf-call-iptables = 1
          net.bridge.bridge-nf-call-ip6tables = 1
          net.ipv4.ip_forward = 1

  - name: update ca cert
    shell: "update-ca-trust extract;sysctl --system"

  - name: start services
    service:
      name: "{{item}}"
      state: restarted
      enabled: yes
    loop:
      - docker
      - systemd-modules-load.service

六.利用ansible安装cri-dockerd

$devops@harbor ansible$ $ tar zxf cri-dockerd-0.4.4.amd64.tgz

$devops@harbor ansible$ $ ls

ansible.cfg cri-dockerd cri-dockerd-0.4.4.amd64.tgz inventory kubernetes-devops.yml

$devops@harbor ansible$ $ cd cri-dockerd/

$devops@harbor cri-dockerd$ $ ls

cri-dockerd cri-docker.service cri-docker.socket

$devops@harbor cri-dockerd$ $ vim cri-docker.service
$Service$
Type=notify

ExecStart=/usr/bin/cri-dockerd --container-runtime-endpoint fd:// --network-plugin=cni --pod-infra-container-image=reg.timinglee.org/k8s/pause:3.10.1

ExecReload=/bin/kill -s HUP $MAINPID

TimeoutSec=0

RestartSec=2

Restart=always

$devops@harbor ansible$ $ vim kubernetes-devops.yml

name: devops k8s from ansible

hosts: all

tasks:

name: setup docker repo

yum_repository:

name: docker

description: docker

baseurl: http://172.25.254.11:4444/docker

file: docker

gpgcheck: no

name: install docker

dnf:

name: docker-ce

state: present

name: setup docker.service

replace:

path: /lib/systemd/system/docker.service

regexp: "ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock"

replace: "ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --iptables=true"

name: setup docker registry

copy:

content: |

{

"registry-mirrors": $"https://172.25.254.11"$

}

dest: /etc/docker/daemon.json

name: setup docker certs

file:

path: /etc/docker/certs.d/172.25.254.11

state: directory

name: cp certs file

copy:

src: "{{item.src}}"

dest: "{{item.dest}}"

loop:

src: ./ca.crt

dest: /etc/docker/certs.d/172.25.254.11/ca.crt

src: ./ca.crt

dest: /etc/pki/ca-trust/source/anchors/ca.crt

########################################################

name: stup cri-dockerd

copy:

src: "{{item.src}}"

dest: "{{item.dest}}"

mode: '0755'

loop:

src: ./cri-dockerd/cri-dockerd

dest: /usr/bin/cri-dockerd

src: "./cri-dockerd/cri-docker.service"

dest: "/lib/systemd/system/cri-docker.service"

src: "./cri-dockerd/cri-docker.socket"

dest: "/lib/systemd/system/cri-docker.socket"

name: update ca cert

shell: "update-ca-trust extract;sysctl --system;systemctl daemon-reload"

name: start services

service:

name: "{{item}}"

state: restarted

enabled: yes

loop:

docker

systemd-modules-load.service

cri-docker

七.利用ansible部署k8s软件

name: install kubernetes from ansible

hosts: all

tasks:

name: setup k8s repo

yum_repository:

name: kubernetes

description: kubernetes

baseurl: https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.35/rpm

file: kubernetes

gpgcheck: no

name: install software in master

dnf:

name: "{{item}}"

state: present

loop:

kubeadm-1.35.7-150500.1.1

kubelet-1.35.7-150500.1.1

kubectl-1.35.7-150500.1.1

when: inventory_hostname == "172.25.254.12"

name: install software in work node

dnf:

name: "{{item}}"

state: present

loop:

kubeadm-1.35.7-150500.1.1

kubelet-1.35.7-150500.1.1

when: inventory_hostname == "172.25.254.13" or inventory_hostname == "172.25.254.14"

name: setup kubectl kubeadm complication

lineinfile:

path: /root/.bashrc

line: |

source <(kubectl completion bash)

source <(kubeadm completion bash)

when: inventory_hostname == "172.25.254.12"

name: start kubelet

service:

name: kubelet

state: started

enabled: yes

#让补齐功能生效