docker compose 部署ELK 8.X及监控

1. 前置环境

安装docker

安装docker-compose

相关版本:

ES:8.5.0

kibana:8.5.0

logstash:8.5.0

2. docker-compose yaml文件

shell 复制代码
# 在指定路径创建配置文件
vim docker-compose-es-kibana-logstash.yaml
yaml 复制代码
version: "3"
services:
   elasticsearch: 
     restart: always
     image: docker.elastic.co/elasticsearch/elasticsearch:8.5.0
     container_name: elasticsearch
     hostname: elasticsearch
     network_mode: host
     privileged: true
     ulimits:
      memlock:
        soft: -1
        hard: -1
     environment:
      - "ES_JAVA_OPTS=-Xms8192m -Xmx8192m"
      - "http.host=0.0.0.0"
      - "node.name=es_node01"
      - "cluster.name=es_cluster"
      - "discovery.type=single-node"
     ports:
      - "9200:9200"
      - "9300:9300"
     volumes:
       - "/opt/config/es/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml"  
       - "/opt/config/es/plugins:/usr/share/elasticsearch/plugins:rw"
       - "/opt/data/es/data:/usr/share/elasticsearch/data:rw"
       
   kibana:
    restart: always
    container_name: kibana
    image: docker.elastic.co/kibana/kibana:8.5.0
    network_mode: host
    environment:
      - XPACK_MONITORING_COLLECTION_ENABLED="true"
    ports:
      - "5601:5601"
    volumes:
      - "/opt/config/kibana/config:/usr/share/kibana/config"
      
   logstash:
     image: docker.elastic.co/logstash/logstash:8.5.0
     container_name: logstash
     network_mode: host
     ports:
       - "9600:9600"
     restart: always
     volumes:
       - "/opt/config/logstash/config:/usr/share/logstash/config"
       - "/opt/config/logstash/pipeline:/usr/share/logstash/pipeline"
       - "/opt/data/logstash:/usr/share/logstash/data"
       - "/opt/logs/logstash:/usr/share/logstash/logs"

3. elasticsearch 配置

  1. 数据存储映射路径配置777权限

    shell 复制代码
    chmod -R 777 /opt/data/es/data
  2. es配置文件

    shell 复制代码
    # 根据配置文件映射的路径,创建es配置文件
    vim /opt/config/es/config/elasticsearch.yml
    yaml 复制代码
    # 集群节点名称
    node.name: "es_node01"
    
    # 设置集群名称为elasticsearch
    cluster.name: "es_cluster"
    
    # 网络访问限制
    network.host: 0.0.0.0
    
    # 以单一节点模式启动
    discovery.type: single-node
    
    # 注意:ES 8.X的版本,需要使用 node.roles配置角色,否则启动报错
    #当前该节点是不是有资格竞选主节点
    #node.master: true
    #当前该节点是否存储数据
    #node.data: true
    node.roles: [master,data,remote_cluster_client]
    
    # 是否支持跨域
    http.cors.enabled: true
    
    # 表示支持所有域名
    http.cors.allow-origin: "*"
    
    # 设置映射端口
    http.port: 9200
    
    # 内部节点之间沟通端口
    transport.port: 9300
    
    # 内存交换的选项,官网建议为true
    bootstrap.memory_lock: true
    
    # 修改安全配置、关闭证书校验、启动xpack监控
    xpack.security.http.ssl:
      enabled: false
    xpack.security.transport.ssl:
      enabled: false
    xpack.security.enabled: false
    xpack.monitoring.collection.enabled: true
    
    # http传输内容的最大容量
    http.max_content_length: 200mb

4. logstash 配置

  1. 数据存储映射路径配置777权限

    shell 复制代码
    chmod -R 777 /opt/data/logstash
  2. logstash配置文件

    shell 复制代码
    vim /opt/config/logstash/config/logstash.yml
    yaml 复制代码
    # 开启xpack监控
    xpack.monitoring.enabled: true
    xpack.monitoring.elasticsearch.hosts: ["http://localhost:9200"]
    xpack.monitoring.collection.interval: 10s
  3. 日志文件配置

    shell 复制代码
    vim /opt/config/logstash/config/log4j2.properties
    properties 复制代码
    status = error
    name = LogstashPropertiesConfig
    
    appender.console.type = Console
    appender.console.name = plain_console
    appender.console.layout.type = PatternLayout
    appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c]%notEmpty{[%X{pipeline.id}]}%notEmpty{[%X{plugin.id}]} %m%n
    
    appender.json_console.type = Console
    appender.json_console.name = json_console
    appender.json_console.layout.type = JSONLayout
    appender.json_console.layout.compact = true
    appender.json_console.layout.eventEol = true
    
    rootLogger.level = ${sys:ls.log.level}
    rootLogger.appenderRef.console.ref = ${sys:ls.log.format}_console
  4. pipelines配置文件

    shell 复制代码
    vim /opt/config/logstash/config/pipelines.yml
    yaml 复制代码
    # This file is where you define your pipelines. You can define multiple.
    # For more information on multiple pipelines, see the documentation:
    #   https://www.elastic.co/guide/en/logstash/current/multiple-pipelines.html
    
    - pipeline.id: main
      path.config: "/usr/share/logstash/pipeline"
  5. 配置logstash数据处理流程,依据实际项目需求配置,此处以input为kafka,output为es举例

    shell 复制代码
    vim /opt/config/logstash/pipeline/logstash-kafka-es.conf
    conf 复制代码
    input {
      kafka {
        bootstrap_servers => "localhost:9092"
        topics => ["topic1"]
        codec => "json"
      }
    }
    
    output {
      if [fields][type] == "topic1" {
        elasticsearch {
          hosts => ["http://localhost:9200"]
          index => "topic1-%{+YYYY.MM.dd}"
        }
      }
    }

5. kibana 配置

shell 复制代码
vim /opt/config/kibana/config/kibana.yml
yaml 复制代码
# Default Kibana configuration for docker target
i18n.locale: zh-CN
server.host: "0.0.0.0"
server.shutdownTimeout: "5s"
elasticsearch.hosts: [ "http://localhost:9200" ]
monitoring.ui.container.elasticsearch.enabled: true

#此处为新生成的kibana账号和密码
elasticsearch.username: "kibana"
elasticsearch.password: "123456"

xpack.monitoring.enabled: true
xpack.monitoring.elasticsearch.hosts: ["http://localhost:9200"]
xpack.monitoring.kibana.collection.enabled: true
xpack.monitoring.kibana.collection.interval: 10000

6. docker compose 启动/停止

在docker-compose yaml所在的路径运行启动或停车命令

shell 复制代码
# 启动
docker-compose -f docker-compose-es-kibana-logstash.yaml up -d

# 停止
docker-compose -f docker-compose-es-kibana-logstash.yaml down

7. 访问测试

es:http://ip:9200

kibana:http://ip:5601

kibana 监控集群状态:kibana --> Management --> 堆栈监测

相关推荐
Jolen_xie3 小时前
Ubuntu下用Docker部署群晖系统---Virtual DSM --zerotier实现连接
linux·ubuntu·docker
前端李易安4 小时前
Git中HEAD、工作树和索引的区别
git·elasticsearch·搜索引擎
深度学习真难4 小时前
docker启动kafka、zookeeper、kafdrop
docker·zookeeper·kafka
qq_172805595 小时前
Docker login 报证书存储错误的解决办法
docker·容器·证书
Elastic 中国社区官方博客5 小时前
从 App Search 到 Elasticsearch — 挖掘搜索的未来
大数据·人工智能·elasticsearch·搜索引擎·ai·全文检索·数据库开发
有来技术6 小时前
Linux 服务器安装 Docker - CentOS 9 (Stream)
linux·服务器·docker
生活百般滋味,人生需要笑对。 --佚名6 小时前
docker如何安装redis
redis·docker·容器
Karoku0667 小时前
【docker集群应用】Docker常用命令
运维·数据库·docker·容器
蜗牛丨10 小时前
Go Vue3 CMS管理后台(前后端分离模式)
mysql·docker·go·vue3·axios·gin·jwt·分页·跨域·ant design vue·log·gorm·otp动态码登录·validator·模型绑定·权限判断
菜鸟小灰灰10 小时前
搭建私有docker仓库
运维·docker·容器