存储docker registry的认证信息
建立harbor私有仓库
# docker login reg.timinglee.org
# vim pod-auth.yml
apiVersion: v1
kind: Pod
metadata:
name: test-noauth
spec:
containers:
- name: nginxtest
image: reg.timinglee.org/test/nginx:latest # 无认证
# kubectl create secret docker-registry docker-auth --docker-server=reg.timinglee.org --docker-username=admin --docker-password=lee
# kubectl get secrets 查看结果：
# vim pod-auth.yml
apiVersion: v1
kind: Pod
metadata:
name: test-auth
spec:
containers:
- name: nginxtest
image: reg.timinglee.org/test/nginx:latest
imagePullSecrets: #不设定docker认证时无法下载镜像
- name: docker-auth
volumes配置管理
容器中文件在磁盘上是临时存放的，这给容器中运行的特殊应用程序带来一些问题
当容器崩溃时，kubelet将重新启动容器，文件会丢失，因为容器会以干净的状态重建。
当在一个 Pod 中同时运行多个容器时，常常需要在这些容器之间共享文件。
Kubernetes 卷具有明确的生命周期与使用它的 Pod 相同卷比 Pod 中运行的任何容器的存活期都长，在容器重新启动时数据也会得到保留
当一个 Pod 不再存在时，卷也将不再存在。
Kubernetes 可以支持许多类型的卷，Pod 也能同时使用任意数量的卷。
卷不能挂载到其他卷，也不能与其他卷有硬链接。 Pod 中的每个容器必须独立地指定每个卷的挂载位置。
emptyDir卷
master ~]# mkdir -p volumes
# cd volumes /
# vim empty.yml
apiVersion: v1
kind: Pod
metadata:
labels:
run: empty
name: empty
spec:
containers:
- image: busybox
name: busybox
command:
/bin/sh
-c
sleep 100000
volumeMounts:
- mountPath: /cache
name: cache-vol
- image: nginx
name: nginx
volumeMounts:
- mountPath: /usr/share/nginx/html
name: cache-vol
volumes:
- name: cache-vol
emptyDir:
medium: Memory
sizeLimit: 100Mi
# kubectl apply -f empty.yml
# kubectl get pods -o wide
# curl 10.244.2.44
# kubectl exec -it pods/empty -c busybox -- /bin/sh
/ # ls
/ # cd cache/
/cache # dd if=/dev/zero of=bigfile bs=1M count=99
# curl 10.244.2.44
hostPath
# kubectl run hostpath --image nginx --dry-run=client -o yaml > hostpath.yml
# vim hostpath.yml
apiVersion: v1
kind: Pod
metadata:
labels:
run: hostpath
name: hostpath
spec:
containers:
- image: nginx
name: hostpath
volumeMounts:
- mountPath: /usr/share/nginx/html
name: timinglee
volumes:
- name: timinglee
hostPath:
path: /data
type: DirectoryOrCreate #当/data目录不存在时自动建立
# kubectl get pods -o wide
# curl 10.244.2.45
ssh -l root node1
node1 ~]# ll /data/
master ~]# echo hello timinglee > /data/index.html
nfs卷
建立nfs共享存储
node3 ~]# mkdir /share
# dnf install nfs-utils -y
# systemctl enable --now nfs-server.service
# echo "/share *(sync,rw,no_root_squash)" > /etc/exports
# exportfs -rv
# showmount -e
在所有work节点安装nfs-utils
master ~]# for i in 10 20 ; do ssh -l root 172.25.254.$i dnf install nfs-utils -y ; done
# for i in 10 20; do ssh -l root 172.25.254.$i showmount -e 172.25.254.30 ; done
建立nfs卷
master ~]# kubectl run web --image nginx --dry-run=client -o yaml >> nfs.yml
# vim nfs.yml
apiVersion: v1
kind: Pod
metadata:
labels:
run: web1
name: web1
spec:
nodeName: node1
containers:
- image: nginx
name: web1
volumeMounts:
- mountPath: /usr/share/nginx/html
name: cache-vol
volumes:
- name: cache-vol
nfs:
server: 172.25.254.30
path: /share
# kubectl apply -f nfs.yml
# kubectl get pods -o wide
node3 ~]# echo hello timinglee > /share/index.html
master ~]# kubectl delete -f nfs.yml
# vim nfs.yml
# kubectl get pods -o wide
# curl 10.244.2.46
PersistentVolume的静态持久卷
建立存储目录
node3 ~]# mkdir /share/pv{1..3} -p
# dnf install nfs-utils -y
# systemctl enable --now nfs-server
# echo " /share *(sync,rw,no_root_squash) " > /etc/exports
# exportfs -rv
编写pv建立的yaml文件
master volumes]# vim pv.yml
apiVersion: v1
kind: PersistentVolume
metadata:
name: pv1
spec:
capacity:
storage: 5Gi
volumeMode: Filesystem
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Retain
storageClassName: nfs
nfs:
path: /share/pv1
server: 172.25.254.30
apiVersion: v1
kind: PersistentVolume
metadata:
name: pv2
spec:
capacity:
storage: 10Gi
volumeMode: Filesystem
accessModes:
- ReadWriteMany
persistentVolumeReclaimPolicy: Retain
storageClassName: nfs
nfs:
path: /share/pv2
server: 172.25.254.30
apiVersion: v1
kind: PersistentVolume
metadata:
name: pv3
spec:
capacity:
storage: 15Gi
volumeMode: Filesystem
accessModes:
- ReadOnlyMany
persistentVolumeReclaimPolicy: Retain
storageClassName: nfs
nfs:
path: /share/pv3
server: 172.25.254.30
# kubectl apply -f pv.yml
# kubectl get pv
一次性指定多个pvc
master ~]# vim pv.yml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: pvc1
spec:
storageClassName: nfs
accessModes:
- ReadWriteOnce #写什么取决于上面的 name 写的什么
resources:
requests:
storage: 1Gi
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: pvc2
spec:
storageClassName: nfs
accessModes:
- ReadWriteMany
resources:
requests:
storage: 5Gi
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: pvc3
spec:
storageClassName: nfs
accessModes:
- ReadOnlyMany
resources:
requests:
storage: 10Gi
# kubectl get pv c 查看结果：
# kubectl -n kube-system get pvc
master ~]# vim pvc-test1.yml
apiVersion: v1
kind: Pod
metadata:
name: pvc-test1
spec:
containers:
- name: nginx
image: nginx:latest
volumeMounts:
- name: pv-storage
mountPath: /usr/share/nginx/html
volumes:
- name: pv-storage
persistentVolumeClaim:
claimName: pvc1
# kubectl exec -it pod-pvc1-test1 -- /bin/bash
/ # echo "pvc-test1" > /usr/share/nginx/html/test 1 .txt
docker ]# cat pv1/test1.txt 查看结果：
master ~]# kubectl exec -it pods/pvc-test1 /bin/bash
/# cd /usr/share/nginx/html
html# ls
# vim checkpvpod.yml
apiVersion: v1
kind: Pod
metadata:
name: timinglee
spec:
containers:
- image: nginx
name: nginx
volumeMounts:
- mountPath: /usr/share/nginx/html
name: vol1
volumes:
- name: vol1
persistentVolumeClaim:
claimName: pvc1
# kubectl apply -f checkpvpod.yml
# kubectl get pods
# curl 10.244.4.12
node3 pv1]# echo hello timinglee > index.html
删除pvc和pv
master ~]# kubectl get pv
# kubectl delete pvc pvc2
# kubectl get pv 查看结果：
# kubectl edit pv pv2
动态持久卷
上传所需镜像
volumes]# docker load -i /root/nfs-subdir-external-provisioner-4.0.2.tar
# docker tag registry.k8s.io/sig-storage/nfs-subdir-external-provisioner:v4.0.2 reg.timinglee.org/sig-storage/nfs-subdir-external-provisioner:v4.0.2
# docker push reg.timinglee.org/sig-storage/nfs-subdir-external-provisioner:v4.0.2
# kubectl delete pvc pvc1 ; kubectl delete pvc pvc 2; kubectl delete pvc pvc 3
# kubectl delete pv pv1 ; kubectl delete pv pv 2; kubectl delete pv pv 3
建立授权
# vim storagesa.yml
apiVersion: v1
kind: Namespace
metadata:
name: nfs-client-provisioner
apiVersion: v1
kind: ServiceAccount
metadata:
name: nfs-client-provisioner
namespace: nfs-client-provisioner
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: nfs-client-provisioner-runner
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["create", "update", "patch"]
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: run-nfs-client-provisioner
subjects:
- kind: ServiceAccount
name: nfs-client-provisioner
namespace: nfs-client-provisioner
roleRef:
kind: ClusterRole
name: nfs-client-provisioner-runner
apiGroup: rbac.authorization.k8s.io
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: leader-locking-nfs-client-provisioner
namespace: nfs-client-provisioner
rules:
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "list", "watch", "create", "update", "patch"]
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: leader-locking-nfs-client-provisioner
namespace: nfs-client-provisioner
subjects:
- kind: ServiceAccount
name: nfs-client-provisioner
namespace: nfs-client-provisioner
roleRef:
kind: Role
name: leader-locking-nfs-client-provisioner
apiGroup: rbac.authorization.k8s.io
vim storagesa.yml
apiVersion: v1
kind: Namespace
metadata:
name: nfs-client-provisioner
apiVersion: v1
kind: ServiceAccount
metadata:
name: nfs-client-provisioner
namespace: nfs-client-provisioner
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: nfs-client-provisioner-runner
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["create", "update", "patch"]
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: run-nfs-client-provisioner
subjects:
- kind: ServiceAccount
name: nfs-client-provisioner
namespace: nfs-client-provisioner
roleRef:
kind: ClusterRole
name: nfs-client-provisioner-runner
apiGroup: rbac.authorization.k8s.io
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: leader-locking-nfs-client-provisioner
namespace: nfs-client-provisioner
rules:
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "list", "watch", "create", "update", "patch"]
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: leader-locking-nfs-client-provisioner
namespace: nfs-client-provisioner
subjects:
- kind: ServiceAccount
name: nfs-client-provisioner
namespace: nfs-client-provisioner
roleRef:
kind: Role
name: leader-locking-nfs-client-provisioner
apiGroup: rbac.authorization.k8s.io
# kubectl -n nfs-client-provisioner get sa
建立控制器
# vim storageclassdep.yml
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
app: nfs-client-provisioner
template:
metadata:
labels:
app: nfs-client-provisioner
spec:
serviceAccountName: nfs-client-provisioner
containers:
- name: nfs-client-provisioner
image: sig-storage/nfs-subdir-external-provisioner:v4.0.2
volumeMounts:
- name: nfs-client-root
mountPath: /persistentvolumes
env:
- name: PROVISIONER_NAME
value: k8s-sigs.io/nfs-subdir-external-provisioner
- name: NFS_SERVER
value: 172.25.254.30
- name: NFS_PATH
value: /share
volumes:
- name: nfs-client-root
nfs:
server: 172.25.254.30
path: /share
# kubectl apply -f storageclassdep.yml
# kubectl -n nfs-client-provisioner get pods
建立存储类
# vim storageclass.yml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: nfs-client
provisioner: k8s-sigs.io/nfs-subdir-external-provisioner
parameters:
archiveOnDelete: "false"
# kubectl apply -f storageclass.yml
# kubectl get storageclasses.storage.k8s.io
建立pvc
# vim pvc.yml
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: test-claim
spec:
storageClassName: nfs-client
accessModes:
- ReadWriteMany
resources:
requests:
storage: 1G
# kubectl get pvc
# kubectl delete -f pvc.yml
设定默认存储类
# kubectl edit sc nfs-client
# kubectl get sc
# vim pvc.yml
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: test-claim
spec:
#storageClassName: nfs-client
accessModes:
- ReadWriteMany
resources:
requests:
storage: 1G
# kubectl apply -f pvc.yml
# kubectl get pvc
statfulset控制器整合动态卷
# kubectl create service clusterip timinglee --tcp 80:80 --clusterip="None" --dry-run=client -o yaml > headless.yml statfulset控制器整合动态卷
# vim headless.yml
apiVersion: v1
kind: Service
metadata:
labels:
app: timinglee
name: timinglee
spec:
clusterIP: None
ports:
- name: webport
port: 80
protocol: TCP
targetPort: 80
selector:
app: webserver
type: ClusterIP
# kubectl create deployment webserver --image nginx --replicas 1 --dry-run=client -o yaml > statefulset.yml 创建statefulset
# vim statefulset.yml
apiVersion: apps/v1
kind: StatefulSet
metadata:
labels:
app: webserver
name: webserver
spec:
serviceName: "timinglee"
replicas: 1
selector:
matchLabels:
app: webserver
template:
metadata:
labels:
app: webserver
spec:
containers:
- image: nginx
name: nginx
volumeMounts:
- name: www
mountPath: /usr/share/nginx/html
volumeClaimTemplates:
- metadata:
name: www
spec:
storageClassName: nfs-client
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
# kubectl apply -f statefulset.yml
# kubectl get statefulsets.apps
# kubectl get pods
# kubectl get pvc
# kubectl scale statefulset webserver --replicas 2
# kubectl get pods
# kubectl get pvc
# kubectl scale statefulset webserver --replicas 3
node3 ~]# cd /share/
# echo webserver1 > default-www-webserver-0-pvc- *** /index.html
# echo webserver 2 > default-www-webserver- 1 -pvc- *** /index.html
# echo webserver 3 > default-www-webserver- 2 -pvc- *** /index.html
master ~]# kubectl run -it testpod --image busyboxplus
$ curl webserver-0.timinglee
# kubectl delete -f statefulset.yml
# kubectl apply -f statefulset.yml
fannel插件转换为calico插件
部署calico
# kubectl delete -f /root/kube-flannel.yml 【删除flannel插件】
# rm -rf /etc/cni/net.d/10-flannel.conflist 【删除所有节点上flannel配置文件,避免冲突】
# ssh -l root 172.25.254.10 rm -rf /etc/cni/net.d/10-flannel.conflist
# ssh -l root 172.25.254.20 rm -rf /etc/cni/net.d/10-flannel.conflist
# mkdir -p /root/network
# cd /root/network/
# vim calico.yaml 【修改文件】
# docker load -i calico-3.28.1.tar
# docker tag calico/cni:v3.28.1 reg.timinglee.org/calico/cni:v3.28.1
# docker push reg.timinglee.org/calico/cni:v3.28.1
# docker tag calico/node:v3.28.1 reg.timinglee.org/calico/node:v3.28.1
# docker push reg.timinglee.org/calico/node:v3.28.1
# docker tag calico/kube-controllers:v3.28.1 reg.timinglee.org/calico/kube-controllers:v3.28.1
# docker push reg.timinglee.org/calico/kube-controllers:v3.28.1
# docker tag calico/typha:v3.28.1 reg.timinglee.org/calico/typha:v3.28.1
# docker push reg.timinglee.org/calico/typha:v3.28.1
# kubectl apply -f calico.yaml --validate=false 【运行文件】
# kubectl -n kube-system get pods | grep calico
calico测试
# kubectl run testpod --image nginx
# kubectl get pods
# kubectl get pods -o wide
# curl 10.244.166.128
调度器
nodeName调度
# watch -n 1 kubectl get pods -o wide
# mkdir -p Scheduler
# cd Scheduler /
# kubectl run nginx --image nginx --dry-run=client -o yaml > nginx.yml
# vim nginx.yml
apiVersion: v1
kind: Pod
metadata:
labels:
run: nginx
name: nginx
spec:# nodeName: #未指定调度阶段
containers:
- image: nginx
name: nginx
# kubectl apply -f nginx.yml
# kubectl delete -f nginx.yml
# kubectl apply -f nginx.yml
nodeselector调度
# vim nginx.yml
apiVersion: v1
kind: Pod
metadata:
labels:
run: nginx
name: nginx
spec:
nodeSelector:
app: timinglee
containers:
- image: nginx
name: nginx
# kubectl get nodes --show-labels
# kubectl label nodes
# kubectl label nodes node2 app=timinglee
# kubectl get nodes --show-labels
# kubectl label nodes node2 app-
节点亲和
倾向满足
# vim nginx.yml
apiVersion: v1
kind: Pod
metadata:
labels:
run: nginx
name: nginx
spec:
containers:
- image: nginx
name: nginx
affinity:
nodeAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- preference:
matchExpressions:
- key: disk
operator: In
values:
ssd
iscsi
weight: 50
# kubectl delete -f nginx.yml
# kubectl label nodes node 1 disk=ssd
# kubectl apply -f nginx.yml
# vim nginx.yml
apiVersion: v1
kind: Pod
metadata:
labels:
run: nginx
name: nginx
spec:
containers:
- image: nginx
name: nginx
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
matchExpressions:
key: disk
operator: In
values:
ssd
iscsi
# kubectl apply -f nginx.yml
节点上必须存在标签 disk，并且该标签的值是 ssd 或 iscsi，如果集群中没有节点带有 disk=ssd 或 disk=iscsi 标签，该 Pod 将永远无法调度（状态为 Pending）。
必须满足
# vim nginx.yml
apiVersion: v1
kind: Pod
metadata:
labels:
run: nginx
name: nginx
spec:
containers:
- image: nginx
name: nginx
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
matchExpressions:
key: disk
operator: In
values:
ssd
iscsi
# kubectl apply -f nginx.yml
# vim nginx.yml
apiVersion: v1
kind: Pod
metadata:
labels:
run: nginx
name: nginx
spec:
containers:
- image: nginx
name: nginx
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
matchExpressions:
key: disk
operator: NotIn #反向选择
values:
ssd
iscsi
POD亲和
# kubectl create deployment webcluster --image nginx --replicas 2 --dry-run=client -o yaml > webcluster.yml
# vim webcluster.yml
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: webcluster
name: webcluster
spec:
replicas: 2
selector:
matchLabels:
app: webcluster
template:
metadata:
labels:
app: webcluster
spec:
containers:
- image: nginx
name: nginx
affinity:
podAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: app
operator: In
values:
- webcluster
topologyKey: "kubernetes.io/hostname"
# kubectl apply -f webcluster.yml
POD反亲和
# kubectl create deployment webcluster --image nginx --replicas 2 --dry-run=client -o yaml > webcluster.yml
# vim webcluster.yml
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: webcluster
name: webcluster
spec:
replicas: 3
selector:
matchLabels:
app: webcluster
template:
metadata:
labels:
app: webcluster
spec:
containers:
- image: nginx
name: nginx
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: app
operator: In
values:
- webcluster
topologyKey: "kubernetes.io/hostname"
节点的污点设定
开启监控并启动该一个deployment控制器
# watch -n 1 kubectl get pods -o wide
# kubectl create deployment webcluster --image nginx --replicas 2 --dry-run=client -o yaml > dep.yml
# vim dep.yml
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: webcluster
name: webcluster
spec:
replicas: 2
selector:
matchLabels:
app: webcluster
template:
metadata:
labels:
app: webcluster
spec:
containers:
- image: nginx
name: nginx
# kubectl apply -f dep.yml
设定污点并购观察
NoExecute
# kubectl taint node node1 nodetype=badnode:NoExecute
# kubectl taint node node1 nodetype-
NoSchedule
# kubectl taint node node2 nodetype=badnode:NoSchedule
# kubectl delete -f dep.yml ; kubectl apply -f dep.yml
PreferNoSchedule
# kubectl delete -f dep.yml
# kubectl taint node node2 nodetype=badnode:PreferNoSchedule
# kubectl apply -f dep.yml
# vim webcluster.yml #pod反亲和
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: webcluster
name: webcluster
spec:
replicas: 3
selector:
matchLabels:
app: webcluster
template:
metadata:
labels:
app: webcluster
spec:
containers:
- image: nginx
name: nginx
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: app
operator: In
values:
- webcluster
topologyKey: "kubernetes.io/hostname"
# kubectl apply -f webcluster.yml
污点容忍
设置节点不同类型给的污点
# kubectl taint node node1 name=lee:NoSchedule
# kubectl taint node node2 nodetype=badnode:NoSchedule
运行deployment控制器
# kubectl apply -f dep.yml
# kubectl delete -f dep.yml
精确容忍指定污点
# vim dep.yml
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: webcluster
name: webcluster
spec:
replicas: 2
selector:
matchLabels:
app: webcluster
template:
metadata:
labels:
app: webcluster
spec:
containers:
- image: nginx
name: nginx
tolerations: #污点容忍
- operator: Equal
key: nodetype
value: badnode
effect: NoSchedule
容忍所有标签的NoSchedule污点模式
# vim dep.yml
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: webcluster
name: webcluster
spec:
replicas: 3
selector:
matchLabels:
app: webcluster
template:
metadata:
labels:
app: webcluster
spec:
containers:
- image: nginx
name: nginx
tolerations:
- operator: Exists
effect: NoSchedule
容忍所有污点
# vim dep.yml
kind: Deployment
metadata:
labels:
app: webcluster
name: webcluster
spec:
replicas: 3
selector:
matchLabels:
app: webcluster
template:
metadata:
labels:
app: webcluster
spec:
containers:
- image: nginx
name: nginx
tolerations:
- operator: Exists
# kubectl apply -f dep.yml
ServiceAccount
现在私有仓库时遇到的授权问题
# kubectl run testpod --image nginx --dry-run=client -o yaml > testpod.yaml
# vim testpod.yaml
apiVersion: v1
kind: Pod
metadata:
labels:
run: testpod
name: testpod
spec:
containers:
- image: nginx
name: testpod
# kubectl apply -f testpod.yaml
# kubectl describe pods testpod | grep Service
# vim testpod.yaml 指定使用过私有仓库镜像
apiVersion: v1
kind: Pod
metadata:
labels:
run: testpod
name: testpod
spec:
containers:
- image: reg.timinglee.org/timinglee/myapp:v1 #私有仓库经香港
name: testpod
imagePullPolicy: Always
解决授权问题
# kubectl create serviceaccount timinglee
# kubectl create secret docker-registry docker-login --docker-username admin --docker-password lee --docker-server reg.timinglee.org --docker-email lee@timinglee.org
# kubectl describe sa timinglee
# kubectl edit sa timinglee
# kubectl describe sa timinglee
# vim testpod.yaml
apiVersion: v1
kind: Pod
metadata:
labels:
run: testpod
name: testpod
spec:
serviceAccountName: timinglee
containers:
name: testpod
imagePullPolicy: Always
# kubectl apply -f testpod.yaml
建立集群用户
建立用户证书
# cd /etc/kubernetes/pki/
# openssl genrsa -out timinglee.key 2048
# openssl req -new -key timinglee.key -out timinglee.csr -subj "/CN=timinglee"
# openssl x509 -req -in timinglee.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out timinglee.crt -days 365
# openssl x509 -in timinglee.crt -text -noout 【查看证书】
建立用户
# kubectl config set-credentials timinglee --client-certificate /etc/kubernetes/pki/timinglee.crt --client-key /etc/kubernetes/pki/timinglee.key --embed-certs=true
# kubectl config view
为用户创建集群的安全上下文
# kubectl config set-context timinglee@kubernetes --cluster kubernetes --user timinglee
# kubectl config view 查看结果：
切换用户
# kubectl config use-context timinglee@kubernetes
# kubectl get pods【用户在集群中只有用户身份没有授权】
切换回集群管理
# kubectl config use-context kubernetes-admin@kubernetes
如果需要删除用户
# kubectl config delete-user timinglee
授权
role和rolebinding
# kubectl config use-context kubernetes-admin@kubernetes 切换回管理员
建立role授权
# kubectl create role timingleerole --dry-run=client --verb=get --resource pods -o yaml > timingleerole.yaml
# vim timingleerole.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: timingleerole
rules:
apiGroups:
""
resources:
- pods
verbs:
get
watch
list
create
update
path
delete
apiGroups:
"apps"
resources:
- deployments
verbs:
get
list
watch
create
创建role
# kubectl apply -f myrole.yml
# kubectl describe role myrole
# kubectl create rolebinding timinglee --role myrole --namespace default --user timinglee --dry-run=client -o yaml > rolebinding-myrole.yml
# vim rolebinding-myrole.yml
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: timinglee
namespace: default #角色绑定必须指定namespace
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: myrole
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: timinglee
# kubectl apply -f rolebinding-myrole.yml
# kubectl get rolebindings.rbac.authorization.k8s.io timinglee
切换用户测试授权
# kubectl config use-context timinglee@kubernetes
# kubectl get pods
# kubectl get svc
切换回管理员
# kubectl config use-context kubernetes-admin@kubernetes
clusterrole和clusterrolebind
建立clusterrole
# kubectl create clusterrole timingleeclusterrole --resource=deployment --verb get --dry-run=client -o yaml > timingleeclusterrole.yml
# vim timingleeclusterrole.yml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: timingleeclusterrole
rules:
apiGroups:
""
resources:
- pods
verbs:
get
watch
list
create
update
path
delete
apiGroups:
"apps"
resources:
- deployments
verbs:
get
watch
list
create
apiGroups:
""
resources:
- services
verbs:
get
watch
list
create
# kubectl apply -f timingleeclusterrole.yml
# kubectl describe clusterrole myclusterrole
# vim timingleeclusterrole.yml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: clusterrolebind-myclusterrole
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: myclusterrole
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: timinglee
# kubectl get rolebindings.rbac.authorization.k8s.io timinglee
# kubectl describe clusterrolebindings.rbac.authorization.k8s.io clusterrolebind-myclusterrole
权限测试
# kubectl config use-context timinglee@kubernetes
# kubectl get pods -A 【可以查看pod，也可以建立运行pod】
# kubectl get deployments.apps -A 【集群角色已经授权，可以查看】
# kubectl get svc -A
# kubectl exec -it test-pod -- sh
****# kubectl config use-context kubernetes-admin@kubernetes 【****切换回管理员】
部署helm
helm简介
Helm是Kubernetes 应用的包管理工具，主要用来管理 Charts，类似Linux系统的yum。
Helm Chart是用来封装Kubernetes原生应用程序的一系列YAML文件。可以在你部署应用的时候自定义应用程序的一些 Metadata，以便于应用程序的分发。
对于应用发布者而言：通过Helm打包应用、管理应用依赖关系、管理应用版本并发布应用到软件仓库。
对于使用者而言：使用Helm后能以简单的方式在Kubernetes上查找、安装、升级、回滚、卸载应用程序
安装helm
# tar zxf /root/helm/helm-v3.15.4-linux-amd64.tar.gz
# cd /root/helm/linux-amd64/
# ls
# cp -p helm /usr/local/bin/
配置helm命令补齐
# echo "source <(helm completion bash)" >> ~/.bashrc
# source ~/.bashrc
# helm version 查看结果：
helm常用操作
create 创建一个 chart 并指定名字
dependency 管理 chart 依赖
get 下载一个 release。可用子命令：all、hooks、manifest、notes、values
history 获取 release 历史
install 安装一个 chart
list 列出 release
package 将 chart 目录打包到 chart 存档文件中
pull 从远程仓库中下载 chart 并解压到本地 # helm pull stable/mysql -- untar
repo 添加，列出，移除，更新和索引 chart 仓库。可用子命令：add、index、 list、remove、update
rollback 从之前版本回滚
search 根据关键字搜索 chart。可用子命令：hub、repo
show 查看 chart 详细信息。可用子命令：all、chart、readme、values
status 显示已命名版本的状态
template 本地呈现模板
uninstall 卸载一个 release
upgrade 更新一个 release
version 查看 helm 客户端版本
查询官方应用中心
# helm search hub nginx #在官方仓库中搜索
# helm search repo nginx #在本地仓库中搜索
管理第三方repo源
****# helm repo add aliyun https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts****【添加阿里云】
# helm repo add bitnami https://charts.bitnami.com/bitnami 【添加bitnami仓库】
****# helm repo add miscro http://mirror.azure.cn/kubernetes/charts/****【微软仓库】
# helm repo list 【查看仓库信息】
# helm search repo aliyun 【查看仓库存储helm清单】
# helm repo remove aliyun 【删除第三方存储库】
helm的使用方法
# helm search repo nginx 【查找chart】
# helm show chart bitnami/nginx 【查看chart信息】
安装chart 包
# helm install mariadb miscro/mariadb
# helm list 查看结果：
# kubectl get pods
# helm status mariadb【查看项目的发布状态】
# helm uninstall mariadb【卸载项目】
# helm list 查看结果：
安装项目前预定义项目选项
# helm pull bitnami/nginx
# docker load -i /root/nginx-1.27.1-debian-12-r2.tar
# docker tag bitnami/nginx:1.27.1-debian-12-r2 reg.timinglee.org/bitnami/nginx:1.27.1-debian-12-r2
# docker push reg.timinglee.org/bitnami/nginx:1.27.1-debian-12-r2
# cd ~
# tar zxf nginx-18.1.11.tgz
# cd nginx/
# vim values.yaml 【项目变量文件】
# helm install timinglee /root/nginx
# kubectl get svc
# kubectl get pods 查看结果：
# vim values.yaml 【更新变量文件】
# helm upgrade timinglee .
# kubectl get svc 查看结果：
# kubectl get ingress
# vim /etc/hosts
# curl www.timinglee.org
# helm history timinglee
# helm uninstall timinglee 【删除项目】
# helm list 查看结果：
构建helm中的chart包
Helm Chart目录结构
# helm create timinglee 【简历chart项目】
# tree timinglee/
构建方法
# cd /root/helm/linux-amd64/timinglee/
# vim Chart.yaml
# vim values.yaml
#更改内容
image:
repository: myapp
pullPolicy: IfNotPresent
tag: "v1"
ingress:
enabled: true
className: "nginx"
annotations: {}
kubernetes.io/ingress.class: nginx
kubernetes.io/tls-acme: "true"
hosts:
- host: www.timinglee.org
paths:
- path: /
pathType: ImplementationSpecific
# helm lint . 检测查看：
# helm install timinglee .
# kubectl get deployments.apps
项目打包
# cd /root/helm/linux-amd64
# helm package timinglee/
# ls
项目可以通过各种分享方式发方为任何人后部署即可
# helm install timinglee timinglee-0.1.0.tgz
构建helm仓库
在harbor仓库中构建一个公开的项目
安装helm push插件
如果网络没问题情况下直接安装即可
# cd ~
# dnf install git -y
# helm plugin install https://github.com/chartmuseum/helm-push
离线安装
# mkdir -p ~/.local/share/helm/plugins/helm-push
# tar zxf helm-push_0.10.4_linux_amd64.tar.gz -C ~/.local/share/helm/plugins/helm-push
# ls ~/.local/share/helm/plugins/helm-push
查看helm调用命令是否成功
# helm cm-push --help 查看结果：
上传项目到仓库
# helm repo add timinglee https://reg.timinglee.org/chartrepo/timinglee 【添加仓库】
为helm添加证书
# cp /etc/docker/certs.d/reg.timinglee.org/ca.crt /etc/pki/ca-trust/source/anchors/
# update-ca-trust 【更新本地ca认证库】
再次添加仓库
# helm repo add timinglee https://reg.timinglee.org/chartrepo/timinglee
上传本地项目
命令执行格式：helm cm-push <项目名称> <仓库名称> -u admin -p lee
# helm cm-push timinglee-0.1.0.tgz timinglee -u admin -p lee
查看项目上传情况
# helm search repo timinglee 【上传后数据未更新】
# helm repo update timinglee 【更新仓库】
# helm search repo timinglee 查看结果：
安装项目
# helm install timinglee timinglee/timinglee 【安装】
# curl www.timinglee.org 查看运行：
helm的版本迭代
从新构建新版本项目
# vim /root/helm/linux-amd64/timinglee/Chart.yaml
# vim /root/helm/linux-amd64/timinglee/values.yaml
# helm package timinglee
上传项目到helm仓库中
# helm cm-push timinglee-0.2.0.tgz timinglee -u admin -p lee
# helm repo update timinglee 【更新仓库】
# helm search repo timinglee -l 查看结果：
更新应用
# helm upgrade timinglee timinglee/timinglee
# curl http://www.timinglee.org/
显示项目版本
# helm history timinglee
应用回滚
# helm rollback timinglee
# helm history timinglee
# curl www.timinglee.org 查看运行：
Prometheus
Prometheus架构
在k8s中部署Prometheus
# helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
在helm中添加Prometheus仓库
# mkdir -p prometheus
# cd prometheus/
# tar zxf kube-prometheus-stack-62.6.0.tgz
# ls kube-prometheus-stack/
# tar zxf prometheus-adapter-4.11.0.tgz
# ls prometheus-adapter/
根据所有项目中的values.yaml中指定的image路径下载容器镜像并上传至harbor仓库
容器镜像prometheus 推送
# docker load -i prometheus-62.6.0.tar
# docker tag quay.io/prometheus/prometheus:v2.54.1 reg.timinglee.org/prometheus/prometheus:v2.54.1
# docker tag quay.io/thanos/thanos:v0.36.1 reg.timinglee.org/thanos/thanos:v0.36.1
# docker tag quay.io/prometheus/alertmanager:v0.27.0 reg.timinglee.org/prometheus/alertmanager:v0.27.0
# docker tag quay.io/prometheus-operator/admission-webhook:v0.76.1 reg.timinglee.org/prometheus-operator/admission-webhook:v0.76.1
# docker tag quay.io/prometheus-operator/prometheus-operator:v0.76.1 reg.timinglee.org/prometheus-operator/prometheus-operator:v0.76.1
# docker tag registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20221220-controller-v1.5.1-58-g787ea74b6 reg.timinglee.org/ingress-nginx/kube-webhook-certgen:v20221220-controller-v1.5.1-58-g787ea74b6
# docker tag quay.io/prometheus-operator/prometheus-config-reloader:v0.76.1 reg.timinglee.org/prometheus-operator/prometheus-config-reloader:v0.76.1
# docker push reg.timinglee.org/prometheus/prometheus:v2.54.1
# docker push reg.timinglee.org/thanos/thanos:v0.36.1
# docker push reg.timinglee.org/prometheus/alertmanager:v0.27.0
# docker push reg.timinglee.org/prometheus-operator/admission-webhook:v0.76.1
# docker push reg.timinglee.org/prometheus-operator/prometheus-operator:v0.76.1
# docker push reg.timinglee.org/ingress-nginx/kube-webhook-certgen:v20221220-controller-v1.5.1-58-g787ea74b6
# docker push reg.timinglee.org/prometheus-operator/prometheus-config-reloader:v0.76.1
# docker load -i node-exporter-v1.8.2.tar
# docker tag reg.timinglee.org/prometheus/node-exporter:v1.8.2 reg.timinglee.org/prometheus/node-exporter:v1.8.2
# docker push reg.timinglee.org/prometheus/node-exporter:v1.8.2
容器镜像grafana 推送
# docker load -i grafana-11.2.0.tar
# docker tag grafana/grafana:11.2.0 reg.timinglee.org/grafana/grafana:11.2.0
# docker tag grafana/grafana-image-renderer:latest reg.timinglee.org/grafana/grafana-image-renderer:latest
# docker tag quay.io/kiwigrid/k8s-sidecar:1.27.4 reg.timinglee.org/kiwigrid/k8s-sidecar:1.27.4
# docker tag bats/bats:v1.4.1 reg.timinglee.org/bats/bats:v1.4.1
# docker push reg.timinglee.org/grafana/grafana:11.2.0
# docker push reg.timinglee.org/grafana/grafana-image-renderer:latest
# docker push reg.timinglee.org/kiwigrid/k8s-sidecar:1.27.4
# docker push reg.timinglee.org/bats/bats:v1.4.1
容器镜像nginx-exporter 推送
# docker load -i nginx-exporter-1.3.0-debian-12-r2.tar
# docker tag bitnami/nginx-exporter:1.3.0-debian-12-r2 reg.timinglee.org/bitnami/nginx-exporter:1.3.0-debian-12-r2
# docker push reg.timinglee.org/bitnami/nginx-exporter:1.3.0-debian-12-r2
容器镜像kube-state-metrics 推送
# docker load -i kube-state-metrics-2.13.0.tar
# docker tag registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.13.0 reg.timinglee.org/kube-state-metrics/kube-state-metrics:v2.13.0
# docker tag quay.io/brancz/kube-rbac-proxy:v0.18.0 reg.timinglee.org/brancz/kube-rbac-proxy:v0.18.0
# docker push reg.timinglee.org/kube-state-metrics/kube-state-metrics:v2.13.0
# docker push reg.timinglee.org/brancz/kube-rbac-proxy:v0.18.0
利用helm安装Prometheus
卸载命令（如需要重新安装）: helm -n kube-prometheus-stack uninstall kube-prometheus-stack
# kubectl create namespace kube-prometheus-stack
# cd /root/prometheus/kube-prometheus-stack 【在这个目录执行】
# helm -n kube-prometheus-stack install kube-prometheus-stack .
# kubectl --namespace kube-prometheus-stack get pods 查看结果：
# kubectl -n kube-prometheus-stack get svc
# kubectl -n kube-prometheus-stack edit svc kube-prometheus-stack-grafana 【修改暴漏方式】
各个svc的作用
alertmanager-operated 告警管理
kube-prometheus-stack-grafana 展示prometheus采集到的指标
kube-prometheus-stack-prometheus-node-exporter 收集节点级别的指标的工具
kube-prometheus-stack-prometheus 主程序
登陆grafana
查看grafana密码
# kubectl -n kube-prometheus-stack get secrets kube-prometheus-stack-grafana -o yaml
# echo "YWRtaW4=" | base64 -d
# echo -n "cHJvbS1vcGVyYXRvcg==" | base64 -d
prometheus监控
下载prometheus
官网：https://github.com/prometheus-operator/prometheus-operator
登录使用
设置语言
监控使用示例
建立监控项目
# helm pull bitnami/nginx --version 18.1.11【下载示例所需helm项目】
# vim values.yaml 【修改项目开启监控】
# kubectl -n kube-prometheus-stack edit svc kube-prometheus-stack-prometheus
# kubectl -n kube-prometheus-stack get svc kube-prometheus-stack-prometheus
查看监控
# kubectl -n kube-prometheus-stack get servicemonitors.monitoring.coreos.com --show-labels
# helm install timinglee .【安装项目，在安装之前一定要上传镜像到仓库中】
# kubectl get svc 查看结果：
# ab -c 5 -n 100 http://172.25.254.5 3 /index.html 【压力测试】
