ELK案例 - 技术栈

logstash配置和k8s方式

复制代码

[root@localhost dev]# cat logstash/data/pipeline/logstash.conf 
input {
  beats {
    port => 5044
  }
}
filter {
  mutate {
      remove_field => ["log", "stream", "time", "agent", "input", "host", "ecs"]
  }
  mutate {
    add_field => { "jettech_index_name" => "%{[kubernetes][namespace]}-%{[kubernetes][pod][name]}" }
  }
}
output {
  elasticsearch {
    hosts => ["http://jettech-elasticsearch:9200"]
    index => "%{jettech_index_name}-app-log-%{+YYYY.MM.dd}"
    #user => "elastic"
    #password => "changeme"
  }
}
[root@localhost dev]# cat logstash/data/config/logstash.yml 
http.host: "0.0.0.0"
http.port: 9600
#xpack.monitoring.enabled: true
#xpack.monitoring.elasticsearch.hosts: [ "http://jettech-elasticsearch:9200" ]
#xpack.monitoring.elasticsearch.username: elastic
##xpack.monitoring.elasticsearch.password: changeme
config.reload.automatic: true
config.reload.interval: 3s
pipeline.workers: 2
pipeline.batch.size: 125
pipeline.batch.delay: 50
queue.type: persisted
queue.max_bytes: 4gb
path.logs: /var/log/logstash
pipeline.ecs_compatibility: disabled



[root@wubo deployment]# cat jettech-logstash-deployment-dev.yaml 
apiVersion: v1
kind: Service
metadata:
  labels: {name: jettech-logstash}
  name: jettech-logstash
  namespace: jettech-dev
spec:
  ports:
  - {name: t5044, port: 5044, protocol: TCP, targetPort: t5044}
  selector: {name: jettech-logstash}
  type: ClusterIP
---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels: {name: jettech-logstash}
  name: jettech-logstash
  namespace: jettech-dev
spec:
  replicas: 1
  selector:
    matchLabels: {name: jettech-logstash}
  template:
    metadata:
      labels: {name: jettech-logstash}
      name: jettech-logstash
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: kubernetes.io/hostname
                operator: In
                values:
                - 172.16.11.35
      containers:
      - name: jettech-logstash
        image: harbor.jettech.com/jettechtools/logstash:7.17.27
        env:
        - {name: LS_HEAP_SIZE, value: '1024m'}
        securityContext:
          privileged: true
        ports:
        - {containerPort: 5044, name: t5044, protocol: TCP}
        volumeMounts:
        - name: jettech-logstash-config
          mountPath: /usr/share/logstash/config
        - name: jettech-logstash-data
          mountPath: /usr/share/logstash/data
        - name: jettech-logstash-pipeline
          mountPath: /usr/share/logstash/pipeline
        - name: host-time
          mountPath: /etc/localtime
        imagePullPolicy: Always
      dnsPolicy: ClusterFirstWithHostNet
      restartPolicy: Always
      volumes:
      - name: jettech-logstash-config
        hostPath:
          path: /opt/jettech/dev/logstash/data/config
      - name: jettech-logstash-data
        hostPath:
          path: /opt/jettech/dev/logstash/data/data
      - name: jettech-logstash-pipeline
        hostPath:
          path: /opt/jettech/dev/logstash/data/pipeline
      - name: host-time
        hostPath:
          path: /etc/localtime

es:

复制代码

[root@wubo deployment]# cat jettech-elasticsearch-deployment-dev.yaml 
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: jettech-elasticsearch-cn
  namespace: jettech-dev
  annotations:
    nginx.ingress.kubernetes.io/proxy-body-size: 3096m
spec:
  rules:
  - host: jettech.es.dev.jettech.cn
    http:
      paths:
      - backend:
          service:
            name: jettech-elasticsearch
            port:
              number: 9200
        path: /
        pathType: Prefix
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: jettech-elasticsearch-com
  namespace: jettech-dev
  annotations:
    nginx.ingress.kubernetes.io/proxy-body-size: 3096m
spec:
  rules:
  - host: jettech.es.dev.jettech.com
    http:
      paths:
      - backend:
          service:
            name: jettech-elasticsearch
            port:
              number: 9200
        path: /
        pathType: Prefix
---
apiVersion: v1
kind: Service
metadata:
  labels: {name: jettech-elasticsearch}
  name: jettech-elasticsearch
  namespace: jettech-dev
spec:
  ports:
  - {name: t9200, port: 9200, protocol: TCP, targetPort: t9200}
  - {name: t9300, port: 9300, protocol: TCP, targetPort: t9300}
  selector: {name: jettech-elasticsearch}
  type: ClusterIP
---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels: {name: jettech-elasticsearch}
  name: jettech-elasticsearch
  namespace: jettech-dev
spec:
  replicas: 1
  selector:
    matchLabels: {name: jettech-elasticsearch}
  template:
    metadata:
      labels: {name: jettech-elasticsearch}
      name: jettech-elasticsearch
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: kubernetes.io/hostname
                operator: In
                values:
                - 172.16.11.35
      #tolerations:
      #- key: node-role.jettech.cn/jettech-elasticsearch-dev
      #  operator: Equal
      #  value: "true"
      #  effect: NoSchedule
      #affinity:
      #  nodeAffinity:
      #    requiredDuringSchedulingIgnoredDuringExecution:
      #      nodeSelectorTerms:
      #      - matchExpressions:
      #        - key: node-role.jettech.cn/jettech-elasticsearch-dev
      #          operator: In
      #          values: ["true"]
      containers:
      - name: jettech-elasticsearch
        image: harbor.jettech.com/jettechtools/elasticsearch:7.17.27
        env:
        - {name: cluster.name, value: 'jettech-elasticsearch-cluster'}
        - {name: node.name, value: 'jettech-elasticsearch-node01'}
        - {name: discovery.type, value: 'single-node'}
        - {name: path.repo, value: '/usr/share/elasticsearch/backup'}
        - {name: TZ, value: 'Asia/Shanghai'}
        - {name: ES_JAVA_OPTS, value: '-Xms16g -Xmx16g'}
        - {name: ingest.geoip.downloader.enabled, value: 'false'}
        - {name: http.cors.enabled, value: 'true'}
        - {name: http.cors.allow-origin, value: '*'}
        - {name: http.cors.allow-headers, value: 'Authorization,X-Requested-With,Content-Length,Content-Type'}
        #- {name: network.host, value: '0.0.0.0'}
        #- {name: http.port, value: '9200'}
        #- {name: transport.port, value: '9300'}
        #- {name: bootstrap.memory_lock, value: 'true'}
        #- {name: bootstrap.system_call_filter, value: 'false'}
        #- {name: xpack.security.enabled, value: 'true'}
        #- {name: xpack.security.transport.ssl.enabled, value: 'true'}
        #- {name: xpack.license.self_generated.type, value: 'basic'}
        #- {name: xpack.security.transport.ssl.verification_mode , value: 'certificate'}
        #- {name: xpack.security.transport.ssl.keystore.path, value: 'elastic-certificates.p12'}
        #- {name: xpack.security.transport.ssl.truststore.path, value: 'elastic-certificates.p12'}
        securityContext:
          privileged: true
        ports:
        - {containerPort: 9200, name: t9200, protocol: TCP}
        - {containerPort: 9300, name: t9300, protocol: TCP}
        resources:
          requests:
            cpu: "4"
            memory: 16G
          limits:
            cpu: "8"
            memory: 32G
        volumeMounts:
        - name: jettech-elasticsearch-data
          mountPath: /usr/share/elasticsearch/data
        - name: jettech-elasticsearch-data-backup
          mountPath: /usr/share/elasticsearch/backup
        - name: host-time
          mountPath: /etc/localtime
        imagePullPolicy: Always
      dnsPolicy: ClusterFirstWithHostNet
      restartPolicy: Always
      volumes:
      - name: jettech-elasticsearch-data
        hostPath:
          path: /opt/jettech/dev/elasticsearch/data
      - name: jettech-elasticsearch-data-backup
        hostPath:
          path: /opt/jettech/dev/elasticsearch/backup
      - name: host-time
        hostPath:
          path: /etc/localtime

filebeat

复制代码

[root@wubo deployment]# cat jettech-filebeat-deployment-dev.yaml 
apiVersion: v1
kind: ConfigMap
metadata:
  name: jettech-filebeat-config
  namespace: jettech-dev
data:
  filebeat.yml: |-
    filebeat.inputs:
    - type: container
      paths:
        - /var/log/containers/*.log
      parsers:
        - ndjson:
            keys_under_root: true
            overwrite_destination: true
            add_error_key: true
          container.id_path_from_filename: true
      multiline.type: pattern
      multiline.pattern: '^\d{4}-\d{2}-\d{2}|^\['
      multiline.negate: true
      multiline.match: after
    processors:
      - add_kubernetes_metadata:
          host: ${NODE_NAME}
          matchers:
            - logs_path:
                logs_path: /var/log/containers/

      - drop_event:
          when:
            not:
              or:
                - equals: {kubernetes.namespace: "jettong-dev"}
                - equals: {kubernetes.namespace: "jettodata-prod"}
                #- equals: {kubernetes.namespace: "kube-system"}
                #- equals: {kubernetes.namespace: "cattle-system"}
                #- equals: {kubernetes.namespace: "ingress-nginx"}
                #- equals: {kubernetes.namespace: "cattle-prometheus"}
                #- equals: {kubernetes.namespace: "kube-public"}
                #- equals: {kubernetes.namespace: "kube-node-lease"}
    #output.console:
    #  pretty: true
    output.logstash:
      hosts: ["jettech-logstash:5044"]
      worker: 2
      bulk_max_size: 200
      compression_level: 1
      timeout: 30s
    path.data: /usr/share/filebeat/data
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  labels: {name: jettech-filebeat}
  name: jettech-filebeat
  namespace: jettech-dev
spec:
  selector:
    matchLabels: {name: jettech-filebeat}
  template:
    metadata:
      labels: {name: jettech-filebeat}
    spec:
      containers:
      - name: jettech-filebeat
        image: harbor.jettech.com/jettechtools/filebeat:7.17.27
        env:
        - {name: NODE_NAME, valueFrom: {fieldRef: {fieldPath: spec.nodeName}}}
        securityContext:
          runAsUser: 0
          privileged: true
        volumeMounts:
        - name: config
          mountPath: /usr/share/filebeat/filebeat.yml
          subPath: filebeat.yml
        - name: data
          mountPath: /usr/share/filebeat/data
        - name: varlog
          mountPath: /var/log
          readOnly: true
        - name: dockercontainers
          mountPath: /var/lib/docker/containers
          readOnly: true
        - name: localtime
          mountPath: /etc/localtime
      volumes:
      - name: config
        configMap:
          name: jettech-filebeat-config
      - name: data
        hostPath:
          path: /var/lib/filebeat-data
          type: DirectoryOrCreate
      - name: varlog
        hostPath:
          path: /var/log
      - name: dockercontainers
        hostPath:
          path: /var/lib/docker/containers
      - name: localtime
        hostPath:
          path: /etc/localtime
      serviceAccountName: jettech-filebeat
      tolerations:
      - key: node-role.kubernetes.io/master
        effect: NoSchedule
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: jettech-filebeat
  namespace: jettech-dev
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: jettech-filebeat
rules:
- apiGroups: [""]
  resources: [pods, namespaces, nodes]
  verbs: [get, list, watch]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: jettech-filebeat
subjects:
- kind: ServiceAccount
  name: jettech-filebeat
  namespace: jettech-dev
roleRef:
  kind: ClusterRole
  name: jettech-filebeat
  apiGroup: rbac.authorization.k8s.io
[root@wubo deployment]#

kibana

复制代码

[root@wubo deployment]# cat jettech-kibana-deployment-dev.yaml 
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: jettech-kibana-com
  namespace: jettech-dev
  annotations:
    nginx.ingress.kubernetes.io/proxy-body-size: 3096m
spec:
  rules:
  - host: jettech.kibana.dev.jettech.com
    http:
      paths:
      - backend:
          service:
            name: jettech-kibana
            port:
              number: 5601
        path: /
        pathType: Prefix
---
apiVersion: v1
kind: Service
metadata:
  labels: {name: jettech-kibana}
  name: jettech-kibana
  namespace: jettech-dev
spec:
  ports:
  - {name: t5601, port: 5601, protocol: TCP, targetPort: t5601}
  selector: {name: jettech-kibana}
  type: ClusterIP
  #type: NodePort
  #externalTrafficPolicy: Cluster #Local 
---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels: {name: jettech-kibana}
  name: jettech-kibana
  namespace: jettech-dev
spec:
  replicas: 1
  selector:
    matchLabels: {name: jettech-kibana}
  template:
    metadata:
      labels: {name: jettech-kibana}
      name: jettech-kibana
    spec:
      containers:
      - name: jettech-kibana
        image: harbor.jettech.com/jettechtools/kibana:7.17.27
        env:
        - {name: ELASTICSEARCH_HOSTS, value: 'http://jettech-elasticsearch:9200'}
        #- {name: ELASTICSEARCH_USERNAME, value: 'kibana_system'}
        #- {name: ELASTICSEARCH_PASSWORD, value: '123456aA'}
        securityContext:
          privileged: true
        ports:
        - {containerPort: 5601, name: t5601, protocol: TCP}
        volumeMounts:
        - name: host-time
          mountPath: /etc/localtime
        imagePullPolicy: Always
      dnsPolicy: ClusterFirstWithHostNet
      restartPolicy: Always
      volumes:
      - name: host-time
        hostPath:
          path: /etc/localtime
[root@wubo deployment]#