elasticsearch-curator: es索引生命周期(关闭、删除索引)

1,下载安装

  • rpm包下载安装
bash 复制代码
# 防止安装rpm包报错:error: [upel]: elasticsearch-curator NOKEY ; 
#                  error: [upel]: elasticsearch-curator signature check fail
rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch #参考https://www.elastic.co/guide/en/elasticsearch/reference/7.17/rpm.html

rpm -ivh https://packages.elastic.co/curator/5/centos/7/Packages/elasticsearch-curator-5.8.4-1.x86_64.rpm

2, 配置使用

配置

bash 复制代码
TEST1:/root # cat action.yml
---
actions:
  1:
    action: close
    description: >-
      关闭指定索引,5天前的:test_logstash_
    options:
      skip_flush: False
      delete_aliases: False
      ignore_sync_failures: True
      ignore_empty_list: True
      disable_action: False
    filters:
    - filtertype: pattern
      kind: prefix
      #value: test_logstash_2023_11_12
      value: test_logstash_
    - filtertype: age
      source: name
      direction: older
      #timestring: '%Y.%m.%d'
      timestring: '%Y_%m_%d'
      unit: days
      unit_count: 5

  2:
    action: close
    description: >-
      关闭指定索引,10天前的:^(\w+)_logstash_.*$
    options:
      skip_flush: False
      delete_aliases: False
      ignore_sync_failures: True
      ignore_empty_list: True
      disable_action: False
    filters:
    - filtertype: pattern
      kind: regex
      #value: logstash-
      #value: test_logstash_2023_11_12
      value: '^(\w+)_logstash_.*$'
    - filtertype: age
      source: name
      direction: older
      #timestring: '%Y.%m.%d'
      timestring: '%Y_%m_%d'
      unit: days
      unit_count: 10
      
  3:
    action: delete_indices
    description: >-
      删除过期30天的索引: 按天存储
    options:
      ignore_empty_list: True
      continue_if_exception: True
      disable_action: False
    filters:
    - filtertype: pattern
      kind: regex
      #value: test_logstash_2023_11_12
      value: '^(\w+)_logstash_.*$'
    - filtertype: age
      source: name
      direction: older
      #timestring: '%Y.%m.%d'
      timestring: '%Y_%m_%d'
      unit: days
      unit_count: 31

  4:
    action: delete_indices
    description: >-
      删除过期30天的索引: 按月存储
    options:
      ignore_empty_list: True
      continue_if_exception: True
      disable_action: False
    filters:
    - filtertype: pattern
      kind: regex
      #value: logstash-
      #value: test_logstash_2023_11_12
      value: '^(\w+)_logstash_(\d{4}\S\d{2})$'
    - filtertype: age
      source: name
      direction: older
      #timestring: '%Y.%m.%d'
      timestring: '%Y_%m_%d'
      unit: days
      unit_count: 60

TEST1:/root # cat curator/curatorConfig.yaml 
client:
  hosts:
    - 192.168.0.106
    - 192.168.0.107
    - 192.168.0.108
  port: 9200
  url_prefix:
  use_ssl: False
  certificate:
  client_cert:
  client_key:
  ssl_no_validate: False
  username: elastic
  password: elk123456
  timeout: 60
  master_only: False

logging:
  loglevel: INFO
  logfile: /applog/elk/curator/curator.log
  logformat: default
  blacklist: ['elasticsearch', 'urllib3']

使用

  • curator --config 全局配置文件 action动作配置文件
bash 复制代码
TEST1:/root # curator --config curator/curatorConfig.yaml  action.yml  


TEST1:/root #  tail  /applog/elk/curator/curator.log -f
2024-01-02 11:19:02,339 INFO      Preparing Action ID: 1, "close"
2024-01-02 11:19:02,339 INFO      Creating client object and testing connection
2024-01-02 11:19:02,341 INFO      Instantiating client object
2024-01-02 11:19:02,342 INFO      Testing client connectivity
2024-01-02 11:19:02,345 INFO      Successfully created Elasticsearch client object with provided settings
2024-01-02 11:19:02,348 INFO      Trying Action ID: 1, "close": 关闭指定索引,5天前的:test_logstash_
2024-01-02 11:19:04,088 INFO      Skipping action "close" due to empty list: <class 'curator.exceptions.NoIndices'>
2024-01-02 11:19:04,089 INFO      Action ID: 1, "close" completed.

2024-01-02 11:19:04,089 INFO      Preparing Action ID: 2, "close"
2024-01-02 11:19:04,089 INFO      Creating client object and testing connection
2024-01-02 11:19:04,089 INFO      Instantiating client object
2024-01-02 11:19:04,090 INFO      Testing client connectivity
2024-01-02 11:19:04,095 INFO      Successfully created Elasticsearch client object with provided settings
2024-01-02 11:19:04,097 INFO      Trying Action ID: 2, "close": 关闭指定索引,10天前的:^(\w+)_logstash_.*$
2024-01-02 11:19:04,610 INFO      Closing 30 selected indices: [ 'test_net_logstash_2023_12_15', 'metric_net_logstash_2023_12_16', 'mytest_logstash_2023_12_15',... ]
2024-01-02 11:19:06,240 INFO      Action ID: 2, "close" completed.

2024-01-02 13:49:45,975 INFO      Preparing Action ID: 3, "delete_indices"
2024-01-02 13:49:45,975 INFO      Creating client object and testing connection
2024-01-02 13:49:45,975 INFO      Instantiating client object
2024-01-02 13:49:45,976 INFO      Testing client connectivity
2024-01-02 13:49:45,978 INFO      Successfully created Elasticsearch client object with provided settings
2024-01-02 13:49:45,981 INFO      Trying Action ID: 3, "delete_indices": 删除过期30天的索引
2024-01-02 13:49:46,218 INFO      Deleting 8 selected indices: ['test123_logstash_2023_12_01', 'metric_logstash_2023_12_02', 'monitor_logstash_2023_12_02', ...]
2024-01-02 13:49:46,218 INFO      ---deleting index test123_logstash_2023_12_01
2024-01-02 13:49:46,218 INFO      ---deleting index metric_logstash_2023_12_02
2024-01-02 13:49:46,218 INFO      ---deleting index monitor_logstash_2023_12_02
2024-01-02 13:49:47,475 INFO      Action ID: 3, "delete_indices" completed.

2024-01-02 13:50:49,047 INFO      Trying Action ID: 4, "delete_indices": 删除过期60天的索引: 按月存储
2024-01-02 13:50:49,326 INFO      Deleting 22 selected indices: ['test1_logstash_2023_11', 'test2_logstash_2023_12', 'test3_logstash_2023_11'...]
2024-01-02 13:50:49,326 INFO      ---deleting index test1_logstash_2023_11
2024-01-02 13:50:49,326 INFO      ---deleting index test2_logstash_2023_12
2024-01-02 13:50:49,326 INFO      ---deleting index test3_logstash_2023_11
...
2024-01-02 13:50:56,240 INFO      Job completed.
相关推荐
永洪科技6 小时前
永洪科技荣获商业智能品牌影响力奖,全力打造”AI+决策”引擎
大数据·人工智能·科技·数据分析·数据可视化·bi
weixin_307779136 小时前
Hive集群之间迁移的Linux Shell脚本
大数据·linux·hive·bash·迁移学习
大只鹅7 小时前
解决 Spring Boot 对 Elasticsearch 字段没有小驼峰映射的问题
spring boot·后端·elasticsearch
上海锝秉工控9 小时前
防爆拉线位移传感器:工业安全的“隐形守护者”
大数据·人工智能·安全
cv高级工程师YKY9 小时前
SRE - - PV、UV、VV、IP详解及区别
大数据·服务器·uv
bxlj_jcj10 小时前
深入Flink核心概念:解锁大数据流处理的奥秘
大数据·flink
云资源服务商10 小时前
阿里云Flink:开启大数据实时处理新时代
大数据·阿里云·云计算
Aurora_NeAr11 小时前
Spark SQL架构及高级用法
大数据·后端·spark
王小王-12311 小时前
基于Hadoop的公共自行车数据分布式存储和计算平台的设计与实现
大数据·hive·hadoop·分布式·hadoop公共自行车·共享单车大数据分析·hadoop共享单车
数据与人工智能律师12 小时前
数字资产革命中的信任之锚:RWA法律架构的隐形密码
大数据·网络·人工智能·云计算·区块链