gitlab runner operator部署配置

背景说明

由于公司管理的git runner资源不足，导致并发的任务比较多时，出现大面积的排队，比较影响效率。基于此问题，我们可以自建一部分Runner给到相应的仓库使用。这里我们有自建的

在k8s集群自建Runner

参考文档

先决条件

• Kubernetes v1.21 及更高版本
• Cert manager v1.7.1

安装配置Operator

1. 安装OLM（Operator Lifecycle Manager）

curl -sL https://github.com/operator-framework/operator-lifecycle-manager/releases/download/v0.32.0/install.sh | bash -s v0.32.0
# 如果下载了install.sh，可以直接执行 chmod +x install.sh && ./install.sh v0.32.0
root@yuhaohao-10-10-7-19:~# kubectl  get pod -n olm
NAME                                READY   STATUS    RESTARTS   AGE
catalog-operator-55894c8cf8-d84vw   1/1     Running   0          3d
olm-operator-6946d4b877-t55w6       1/1     Running   0          3d
operatorhubio-catalog-nvq9q         1/1     Running   0          17h
packageserver-64fdd96cbb-ggpnh      1/1     Running   0          3d
packageserver-64fdd96cbb-s7r59      1/1     Running   0          3d

2.安装Operator

kubectl create -f https://operatorhub.io/install/gitlab-runner-operator.yaml

3.查看安装的资源

root@yuhaohao-10-10-7-19:~# kubectl  get csv -n operators
NAME                             DISPLAY         VERSION   REPLACES                         PHASE
gitlab-runner-operator.v1.37.0   GitLab Runner   1.37.0    gitlab-runner-operator.v1.36.0   Succeeded
root@yuhaohao-10-10-7-19:~# kubectl  get pod -n operators
NAME                                                             READY   STATUS    RESTARTS   AGE
gitlab-runner-gitlab-runnercontroller-manager-84d6f69dc8-7nsdc   2/2     Running   0          3d

安装配置Gitlab Runner

1.安装minio

# 这里为了实现gitlab runner的缓存，提高构建效率，可以部署minio作为gitlab runner的缓存
# 这里我们安装的是minio 12.9.0版本
# 这里我们可以基于主机目录，配置StorageClass，实现动态PVC存储
# 配置local-path-provisioner
root@yuhaohao-10-10-7-19:~/minio-install# cat /home/yuhaohao/local-path.yaml
apiVersion: v1
kind: Namespace
metadata:
  name: local-path-storage

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: local-path-provisioner-service-account
  namespace: local-path-storage

---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: local-path-provisioner-role
  namespace: local-path-storage
rules:
  - apiGroups: [""]
    resources: ["pods"]
    verbs: ["get", "list", "watch", "create", "patch", "update", "delete"]

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: local-path-provisioner-role
rules:
  - apiGroups: [""]
    resources: ["nodes", "persistentvolumeclaims", "configmaps", "pods", "pods/log"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "create", "patch", "update", "delete"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["create", "patch"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]

---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: local-path-provisioner-bind
  namespace: local-path-storage
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: local-path-provisioner-role
subjects:
  - kind: ServiceAccount
    name: local-path-provisioner-service-account
    namespace: local-path-storage

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: local-path-provisioner-bind
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: local-path-provisioner-role
subjects:
  - kind: ServiceAccount
    name: local-path-provisioner-service-account
    namespace: local-path-storage

---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: local-path-provisioner
  namespace: local-path-storage
spec:
  replicas: 1
  selector:
    matchLabels:
      app: local-path-provisioner
  template:
    metadata:
      labels:
        app: local-path-provisioner
    spec:
      serviceAccountName: local-path-provisioner-service-account
      containers:
        - name: local-path-provisioner
          image: registry.test.com/test/rancher/local-path-provisioner:v0.0.31
          imagePullPolicy: IfNotPresent
          command:
            - local-path-provisioner
            - --debug
            - start
            - --config
            - /etc/config/config.json
          volumeMounts:
            - name: config-volume
              mountPath: /etc/config/
          env:
            - name: POD_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
            - name: CONFIG_MOUNT_PATH
              value: /etc/config/
      volumes:
        - name: config-volume
          configMap:
            name: local-path-config

---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: local-path
provisioner: rancher.io/local-path
volumeBindingMode: WaitForFirstConsumer
reclaimPolicy: Delete

---
kind: ConfigMap
apiVersion: v1
metadata:
  name: local-path-config
  namespace: local-path-storage
data:
  config.json: |-
    {
            "nodePathMap":[
            {
                    "node":"DEFAULT_PATH_FOR_NON_LISTED_NODES",
                    "paths":["/data"]
            }
            ]
    }
  setup: |-
    #!/bin/sh
    set -eu
    mkdir -m 0777 -p "$VOL_DIR"
  teardown: |-
    #!/bin/sh
    set -eu
    rm -rf "$VOL_DIR"
  helperPod.yaml: |-
    apiVersion: v1
    kind: Pod
    metadata:
      name: helper-pod
    spec:
      priorityClassName: system-node-critical
      tolerations:
        - key: node.kubernetes.io/disk-pressure
          operator: Exists
          effect: NoSchedule
      containers:
      - name: helper-pod
        image: registry.test.com/test/busybox
        imagePullPolicy: IfNotPresent
$ kubectl apply -f local-path.yaml
$ kubectl get pod -n local-path-storage
NAME                                      READY   STATUS    RESTARTS   AGE
local-path-provisioner-6bf6d4456b-f9sbr   1/1     Running   0          3d
$ kubectl  get sc
NAME         PROVISIONER             RECLAIMPOLICY   VOLUMEBINDINGMODE      ALLOWVOLUMEEXPANSION   AGE
local-path   rancher.io/local-path   Delete          WaitForFirstConsumer   false                  16d

# 创建minio所需的pvc
$ cat gitlab-runner-minio-pvc.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  labels:
    app.kubernetes.io/instance: minio
    app.kubernetes.io/managed-by: Helm
  name: gitlab-runner-minio-pvc
  namespace: gitlab-runner
spec:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 100Gi
  storageClassName: local-path
  volumeMode: Filesystem
$ kubectl apply -f gitlab-runner-minio-pvc.yaml
$ kubectl get pvc -n gitlab-runner
NAME                      STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   VOLUMEATTRIBUTESCLASS   AGE
gitlab-runner-minio-pvc   Bound    pvc-31430c05-16cc-4441-814b-7aae70b7b134   100Gi      RWO            local-path     <unset>                 15d

# 部署minio
$ cat minio-values.yaml
root@yuhaohao-10-10-7-19:~/minio-install# cat minio-values.yaml
image:
  registry: registry.test.com
  repository: test/bitnami/minio
  tag: 2023.11.1-debian-11-r0
  pullPolicy: IfNotPresent

fullnameOverride: "minio"
auth:
  rootUser: admin
  rootPassword: "minio12345"
defaultBuckets: "gitlab"
persistence:
  ## @param persistence.enabled Enable MinIO&reg; data persistence using PVC. If false, use emptyDir
  ##
  enabled: true
  storageClass: ""
  ## @param persistence.mountPath Data volume mount path
  mountPath: /bitnami/minio/data
  accessModes:
    - ReadWriteOnce
  size: 100Gi
  ## @param persistence.annotations Annotations for the PVC
  ##
  annotations: {}
  ## @param persistence.existingClaim Name of an existing PVC to use (only in `standalone` mode)
  ##
  existingClaim: "gitlab-runner-minio-pvc"

$ helm install -f minio-values.yaml minio ./minio -n gitlab-runner
root@yuhaohao-10-10-7-19:~/minio-install# kubectl  get pod -n gitlab-runner
NAME                              READY   STATUS    RESTARTS   AGE
minio-6d947866f5-rhmwz            1/1     Running   0          3d

2.安装gitlab runner

# 配置通用的runner配置信息
# 注意pre_clone_script 这里的配置是为了解决`Git LFS is not enabled on this GitLab server`的报错
$ cat custom.yaml
apiVersion: v1
data:
  config.toml: |-
    [[runners]]
      environment = ["GIT_LFS_SKIP_SMUDGE=1"]
      pre_clone_script = """
        echo "Setting up secrets"
        ################## run in runner-helper
        echo "######### ${GITLAB_USER_LOGIN}"
        git config --global lfs.url "https://lfs.test.com"
        git config --global credential.helper store
        echo "https://test:[email protected]" >> $HOME/.git-credentials
        mkdir -pv $HOME/.ssh >> /dev/null
        chmod 700 $HOME/.ssh
        echo -e "Host gitlab.testcom\n\tStrictHostKeyChecking no\n" >> $HOME/.ssh/config
      """
      [runners.kubernetes]
        image_pull_secrets = ["inner"]
        pull_policy = "if-not-present"
        privileged = true
        [runners.kubernetes.node_selector]
          "ci-node" = "True"
        [runners.kubernetes.node_tolerations]
          "ci-node=true" = "NoSchedule"
kind: ConfigMap
metadata:
  name: runner-custom-config
  namespace: gitlab-runner

# 配置minio的secret
$ cat minio-secret.yaml
---
apiVersion: v1
data:
  accesskey: YWRtaW4=
  secretkey: bWluaW8xMjM0NQ==
kind: Secret
metadata:
  name: minio-secret-for-runner
  namespace: gitlab-runner
type: Opaque

# 配置拉取镜像的secrets
$ cat secrets-inner.yaml
apiVersion: v1
data:
  .dockerconfigjson: eyJhdXRocyI6eyjkjkj9ib3QuZG9ja2VyIiwicGFzc3dvcmQiOiJTVGFpMjAyMy5kayIsImF1dGgiOiJkbWx3WlhJdWNtOWliM1F1Wkc5amEyVnlPbE5VWVdreU1ESXpMbVJyIn19fQ==
kind: Secret
metadata:
  name: inner
  namespace: gitlab-runner
type: kubernetes.io/dockerconfigjson

# 配置具体的runner
$ cat test-runner.yaml
---
apiVersion: v1
kind: Secret
metadata:
  name: gitlab-runner-secret-test
  namespace: gitlab-runner
type: Opaque
stringData:
  runner-registration-token: ""
  runner-token: "glrt-t2_73uew3jtestEhDM" # 注意，这里是gitlab runner注册的token

---
apiVersion: apps.gitlab.com/v1beta2
kind: Runner
metadata:
  name: test
  namespace: gitlab-runner
spec:
  logLevel: debug
  # 这里，不要用gitlab-runner-ocp 13.x版本，会有重启runner后，runner无法重新注册连接的问题。
  runnerImage: "registry.test.com/test/gitlab-runner-ocp:v17.4.0"
  gitlabUrl: https://gitlab.test.com/
  buildImage: alpine
  helperImage: registry.test.com/test/gitlab-runner-helper:ubuntu-x86_64-v17.5.1
  concurrent: 50
  token: gitlab-runner-secret-test
  tags: "test"
  locked: true
  cacheType: s3
  cacheShared: true
  s3:
    server: minio:9000
    credentials: minio-secret-for-runner
    bucket: sumo19
    insecure: true
  config: runner-custom-config

$ kubectl apply -f .
# 查看runner
$  kubectl  get pod -n gitlab-runner
NAME                              READY   STATUS    RESTARTS   AGE
minio-6d947866f5-rhmwz            1/1     Running   0          3d1h
test-runner-7475ddc7f-nn7xh   1/1     Running   0          15h

运行测试