redis——岁月云实战

单线程序，基于IO多路复用，基于内存和c语言编写，性能高。redis官方命令

1 数据结构

1.1 key的层级

redis的key可以通过冒号（:）来划分层级，如下图mms:company:order，但系统中可以看到有不少没有的，这些都是不符合开发规则的，研发工程师偷懒。这些事情应该严格要求，否则留下来的人就会很麻烦，根本无法猜到这个key是从哪里来的。

1.2 数据结构

动画解析Redis为什么用跳表而不是红黑树？我看有些面试官就是很无趣，就好像高考阅读的出题人，总以为比写文章的人还理解，人家作者就是因为跳表比红黑树容易实现，性能也不差就用了，搞一堆题目为难面试者，搞得他们也能写出一个redis一样。

2 场景

3 k8s中部署redis集群

3.1 存储器

因为redis数据需要持久化，所以我认为删掉pvc的时候不应该把数据直接删掉，需要手工清理一下，故而单独给他配置了存储器

bash 复制代码

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: managed-redis-storage
provisioner: fuseim.pri/ifs
reclaimPolicy: Retain
mountOptions:
  - hard
  - nfsvers=4

3.2 部署集群

bash 复制代码

apiVersion: v1
kind: Namespace
metadata:
  name: middleware
 
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: redis-configmap
  namespace: middleware
  labels:
    app: redis
data:
  redis.conf: |
    dir "/data"
    maxmemory 0
    maxmemory-policy volatile-lru
    min-slaves-max-lag 5
    min-slaves-to-write 1
    rdbchecksum yes
    rdbcompression yes
    repl-diskless-sync yes
    save 900 1
    save 300 10
    save 60 10000
    maxclients 10000
 
  sentinel.conf: |
    dir "/data"
    sentinel down-after-milliseconds mymaster 10000
    sentinel failover-timeout mymaster 180000
    sentinel parallel-syncs mymaster 5
 
  init.sh: |
    HOSTNAME="$(hostname)"
    INDEX="${HOSTNAME##*-}"
    MASTER="$(redis-cli -h redis -p 26379 sentinel get-master-addr-by-name mymaster | grep -E '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}')"
    MASTER_GROUP="mymaster"
    QUORUM="2"
    REDIS_CONF=/data/conf/redis.conf
    REDIS_PORT=6379
    SENTINEL_CONF=/data/conf/sentinel.conf
    SENTINEL_PORT=26379
    SERVICE=redis-headless
    set -eu
 
    sentinel_update() {
        echo "Updating sentinel config"
        eval MY_SENTINEL_ID="\${SENTINEL_ID_$INDEX}"
        sed -i "1s/^/sentinel myid $MY_SENTINEL_ID\\n/" "$SENTINEL_CONF"
        sed -i "2s/^/sentinel monitor $MASTER_GROUP $1 $REDIS_PORT $QUORUM \\n/" "$SENTINEL_CONF"
        echo "sentinel announce-ip $ANNOUNCE_IP" >> $SENTINEL_CONF
        echo "sentinel announce-port $SENTINEL_PORT" >> $SENTINEL_CONF
    }
 
    redis_update() {
        echo "Updating redis config"
        echo "slaveof $1 $REDIS_PORT" >> "$REDIS_CONF"
        echo "slave-announce-ip $ANNOUNCE_IP" >> $REDIS_CONF
        echo "slave-announce-port $REDIS_PORT" >> $REDIS_CONF
    }
 
    copy_config() {
        cp /readonly-config/redis.conf "$REDIS_CONF"
        cp /readonly-config/sentinel.conf "$SENTINEL_CONF"
    }
 
    setup_defaults() {
        echo "Setting up defaults"
        if [ "$INDEX" = "0" ]; then
            echo "Setting this pod as the default master"
            redis_update "$ANNOUNCE_IP"
            sentinel_update "$ANNOUNCE_IP"
            sed -i "s/^.*slaveof.*//" "$REDIS_CONF"
        else
            DEFAULT_MASTER="$(getent hosts "redis-0.$SERVICE" | awk '{ print $1 }')"
            if [ -z "$DEFAULT_MASTER" ]; then
                echo "Unable to resolve host"
                exit 1
            fi
            echo "Setting default slave config.."
            redis_update "$DEFAULT_MASTER"
            sentinel_update "$DEFAULT_MASTER"
        fi
    }
 
    find_master() {
        echo "Attempting to find master"
        if [ "$(redis-cli -h "$MASTER" ping)" != "PONG" ]; then
           echo "Can't ping master, attempting to force failover"
           if redis-cli -h "$SERVICE" -p "$SENTINEL_PORT" sentinel failover "$MASTER_GROUP" | grep -q 'NOGOODSLAVE' ; then 
               setup_defaults
               return 0
           fi
           sleep 10
           MASTER="$(redis-cli -h $SERVICE -p $SENTINEL_PORT sentinel get-master-addr-by-name $MASTER_GROUP | grep -E '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}')"
           if [ "$MASTER" ]; then
               sentinel_update "$MASTER"
               redis_update "$MASTER"
           else
              echo "Could not failover, exiting..."
              exit 1
           fi
        else
            echo "Found reachable master, updating config"
            sentinel_update "$MASTER"
            redis_update "$MASTER"
        fi
    }
 
    mkdir -p /data/conf/
 
    echo "Initializing config.."
    copy_config
 
    ANNOUNCE_IP=$(getent hosts "redis-$INDEX.$SERVICE" | awk '{ print $1 }')
    if [ -z "$ANNOUNCE_IP" ]; then
        "Could not resolve the announce ip for this pod"
        exit 1
    elif [ "$MASTER" ]; then
        find_master
    else
        setup_defaults
    fi
 
    if [ "${AUTH:-}" ]; then
        echo "Setting auth values"
        ESCAPED_AUTH=$(echo "$AUTH" | sed -e 's/[\/&]/\\&/g');
        sed -i "s/replace-default-auth/${ESCAPED_AUTH}/" "$REDIS_CONF" "$SENTINEL_CONF"
    fi
 
    echo "Ready..."
 
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: redis-probes
  namespace: middleware
  labels:
    app: redis
data:
  check-quorum.sh: |
    #!/bin/sh
    set -eu
    MASTER_GROUP="mymaster"
    SENTINEL_PORT=26379
    REDIS_PORT=6379
    NUM_SLAVES=$(redis-cli -p "$SENTINEL_PORT" sentinel master mymaster | awk '/num-slaves/{getline; print}')
    MIN_SLAVES=1
 
    if [ "$1" = "$SENTINEL_PORT" ]; then
        if redis-cli -p "$SENTINEL_PORT" sentinel ckquorum "$MASTER_GROUP" | grep -q NOQUORUM ; then
            echo "ERROR: NOQUORUM. Sentinel quorum check failed, not enough sentinels found"
            exit 1
        fi
    elif [ "$1" = "$REDIS_PORT" ]; then
        if [ "$MIN_SLAVES" -gt "$NUM_SLAVES" ]; then
            echo "Could not find enough replicating slaves. Needed $MIN_SLAVES but found $NUM_SLAVES"
            exit 1
        fi
    fi
    sh /probes/readiness.sh "$1"
 
  readiness.sh: |
    #!/bin/sh
    set -eu
    CHECK_SERVER="$(redis-cli -p "$1" ping)"
 
    if [ "$CHECK_SERVER" != "PONG" ]; then
        echo "Server check failed with: $CHECK_SERVER"
        exit 1
    fi
 
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: redis
  namespace: middleware
  labels:
    app: redis
 
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: redis
  namespace: middleware
  labels:
    app: redis
rules:
- apiGroups:
    - ""
  resources:
    - endpoints
  verbs:
    - get
 
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: redis
  namespace: middleware
  labels:
    app: redis
subjects:
- kind: ServiceAccount
  name: redis
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: redis
 
---
apiVersion: v1
kind: Service
metadata:
  name: redis-headless
  namespace: middleware
  labels:
    app: redis-ha
  annotations:
    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
spec:
  publishNotReadyAddresses: true
  type: ClusterIP
  clusterIP: None
  ports:
  - name: server
    port: 6379
    protocol: TCP
    targetPort: redis
  - name: sentinel
    port: 26379
    protocol: TCP
    targetPort: sentinel
  selector:
    app: redis-ha
 
---
apiVersion: v1
kind: Service
metadata:
  name: redis
  namespace: middleware
  labels:
    app: redis-ha
  annotations:
spec:
  type: ClusterIP
  ports:
  - name: server
    port: 6379
    protocol: TCP
    targetPort: redis
  - name: sentinel
    port: 26379
    protocol: TCP
    targetPort: sentinel
  selector:
    app: redis-ha
 
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis
  namespace: middleware
  labels:
    app: redis-ha
spec:
  selector:
    matchLabels:
      app: redis-ha
  serviceName: redis-headless
  replicas: 3
  podManagementPolicy: OrderedReady
  updateStrategy:
    type: RollingUpdate
  template:
    metadata:
      labels:
        app: redis-ha
    spec:
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            - labelSelector:
                matchLabels:
                  app: redis-ha
              topologyKey: kubernetes.io/hostname
          preferredDuringSchedulingIgnoredDuringExecution:
            - weight: 100
              podAffinityTerm:
                labelSelector:
                  matchLabels:
                    app:  redis-ha
                topologyKey: failure-domain.beta.kubernetes.io/zone
 
      securityContext:
        fsGroup: 1000
        runAsNonRoot: true
        runAsUser: 1000
 
      serviceAccountName: redis
      imagePullSecrets:
        - name: 密码
      initContainers:
      - name: config-init
        image: 10.101.10.2:8081/mid/redis:7.4.0
        #image: redis:latest  # 此镜像也可用
        imagePullPolicy: IfNotPresent
        resources:
          {}
 
        command:
        - sh
        args:
        - /readonly-config/init.sh
        env:
        - name: SENTINEL_ID_0
          value: 0c09a3866dba0f3b43ef2e383b5dc05980900fd8
 
        - name: SENTINEL_ID_1
          value: e6be0f70406122877338f7c814b17a7c7b648d82
 
        - name: SENTINEL_ID_2
          value: 31f8f52b34feaddcabdd6bf1827aeb02be44d2e3
 
        volumeMounts:
        - name: config
          mountPath: /readonly-config
          readOnly: true
        - name: data
          mountPath: /data
      containers:
      - name: redis
        image: 10.101.10.2:8081/mid/redis:7.4.0
        imagePullPolicy: IfNotPresent
        command:
        - redis-server
        args:
        - /data/conf/redis.conf
        livenessProbe:
          exec:
            command: [ "sh", "/probes/readiness.sh", "6379"]
          initialDelaySeconds: 15
          periodSeconds: 5
        readinessProbe:
          exec:
            command: ["sh", "/probes/readiness.sh", "6379"]
          initialDelaySeconds: 15
          periodSeconds: 5
        resources:
          {}
 
        ports:
        - name: redis
          containerPort: 6379
        volumeMounts:
        - mountPath: /data
          name: data
        - mountPath: /probes
          name: probes
      - name: sentinel
        image: 10.101.10.2:8081/mid/redis:7.4.0
        imagePullPolicy: IfNotPresent
        command:
          - redis-sentinel
        args:
          - /data/conf/sentinel.conf
        livenessProbe:
          exec:
            command: [ "sh", "/probes/readiness.sh", "26379"]
          initialDelaySeconds: 15
          periodSeconds: 5
        readinessProbe:
          exec:
            command: ["sh", "/probes/readiness.sh", "26379"]
          initialDelaySeconds: 15
          periodSeconds: 5
        resources:
          {}
 
        ports:
          - name: sentinel
            containerPort: 26379
        volumeMounts:
        - mountPath: /data
          name: data
        - mountPath: /probes
          name: probes
      volumes:
      - name: config
        configMap:
          name: redis-configmap
      - name: probes
        configMap:
          name: redis-probes
  volumeClaimTemplates:
  - metadata:
      name: data
      annotations:
    spec:
      accessModes:
        - "ReadWriteMany"
      resources:
        requests:
          storage: "5Gi"
      storageClassName: managed-redis-storage

3 Unable to resolve redis-headless.middleware.svc.cluster.local

查看系统日志发现错误日志如下：

bash 复制代码

2024-12-26 05:28:13.161 INFO  com.alibaba.nacos.client.config.impl.ClientWorker - [fixed-dubbo-prod-nacos-headless.middleware.svc.cluster.local_8848] [publish-single] ok, dataId=com.whty.acc.voucher.api.service.VoucherDubboService:::provider:acc-voucher-server, group=dubbo, tenant=dubbo-prod, config={"annotations":[],"canonicalName":"com.whty.acc.voucher.api.service.VoucherDubboService","codeSource...
2024-12-26 10:10:50.045 ERROR org.redisson.connection.SentinelConnectionManager - Unable to resolve redis-headless.middleware.svc.cluster.local
io.netty.resolver.dns.DnsResolveContext$SearchDomainUnknownHostException: Failed to resolve 'redis-headless.middleware.svc.cluster.local' [A(1)] and search domain query for configured domains failed as well: [acc.svc.cluster.local, svc.cluster.local, cluster.local]
        at io.netty.resolver.dns.DnsResolveContext.finishResolve(DnsResolveContext.java:1088)
        at io.netty.resolver.dns.DnsResolveContext.tryToFinishResolve(DnsResolveContext.java:1035)
        at io.netty.resolver.dns.DnsResolveContext.query(DnsResolveContext.java:422)
        at io.netty.resolver.dns.DnsResolveContext.access$700(DnsResolveContext.java:66)

于是对比一下nacos和redis，为什么nacos是好使的，但redis却不行，如下图，出现了差异，那么这个差异是如何造成的呢？

bash 复制代码

[root@master8 ~]# kubectl exec -it acc-voucher-server-7dfc9b49-2t6bd -n acc -- nslookup nacos-headless.middleware.svc.cluster.local
nslookup: can't resolve '(null)': Name does not resolve

Name:      nacos-headless.middleware.svc.cluster.local
Address 1: 10.42.0.29 10-42-0-29.nacos-web-service.middleware.svc.cluster.local
Address 2: 10.42.2.35 10-42-2-35.nacos-web-service.middleware.svc.cluster.local
Address 3: 10.42.1.21 10-42-1-21.nacos-web-service.middleware.svc.cluster.local
[root@master8 ~]# kubectl exec -it acc-voucher-server-7dfc9b49-2t6bd -n acc -- nslookup redis-headless.middleware.svc.cluster.local
nslookup: can't resolve '(null)': Name does not resolve

Name:      redis-headless.middleware.svc.cluster.local
Address 1: 10.42.1.24 10-42-1-24.redis.middleware.svc.cluster.local
Address 2: 10.42.0.39 redis-0.redis-headless.middleware.svc.cluster.local
Address 3: 10.42.2.39 10-42-2-39.redis.middleware.svc.cluster.local