docker中安装es8
1、es8的tar下载地址: tar包下载地址)
2、docker load -i es-name
3、docker run -d --name elasticsearch-8.6.2 -e ES_JAVA_OPTS="-Xms256m -Xmx256m" -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" elasticsearch:8.6.2
4、8开始以后不支持直接使用http方式访问es:
4.1、cd /usr/share/elasticsearch/config
4.2、 vi elasticsearch.yml将xpack.security.enabled: false,或者直接将该配置下的所有enable改为false
解决docker中vi命令无效时修改文件内容麻烦的问题
1、cat > 需要修改的文件名称 << EOF
2、将需要修改的内容全部复制粘 贴到shell中在对内容进行修改, 因为部分内容修改到该文件时该内容将会覆盖原有文件中 是所有内容
3、最后EOF结束修改，修改完成
自动配置类,用于注入es高级客户端及低级客户端用于操作Api
java
@Configuration
public class RestAutoConfigure {
private static final Logger logger = LoggerFactory.getLogger(RestAutoConfigure.class);
@Value("${spring.elasticsearch.rest.uris:}")
private String elasticsearchNodes;
/**
* Description 低级客户端
* @date 2023/8/13 20:11
* @param:
* @return: org.elasticsearch.client.RestClient
*/
@Bean
public RestClient restClient() {
HttpHost[] httpHosts = getHttpHosts(elasticsearchNodes);
if (httpHosts != null && httpHosts.length > 0) {
RestClientBuilder http = RestClient.builder(httpHosts);
return http.build();
}
return null;
}
/**
* Description 阻塞的 Java 客户端
* @date 2023/8/13 20:11
* @param:
* @return: co.elastic.clients.elasticsearch.ElasticsearchClient
*/
@Bean
@Primary
public ElasticsearchClient restHighLevelClient() {
HttpHost[] httpHosts = getHttpHosts(elasticsearchNodes);
if (httpHosts != null && httpHosts.length > 0) {
// Create the RestClient
RestClient restClient = RestClient.builder(httpHosts).build();
// Create the transport with a Jackson mapper
RestClientTransport transport = new RestClientTransport(restClient, new JacksonJsonpMapper());
// create the API client
return new ElasticsearchClient(transport);
}
return null;
}
/**
* Description 获取httpHosts
*
* @date 2022/7/15 9:42
* @param: elasticsearchNodes
* @return: org.apache.http.HttpHost[]
*/
private HttpHost[] getHttpHosts(String elasticsearchNodes) {
if (StrUtil.isNotEmpty(elasticsearchNodes)) {
// 逗号分割多个es路径
String[] elasticsearchNode = elasticsearchNodes.split(",");
HttpHost[] httpHosts = new HttpHost[elasticsearchNode.length];
int index = 0;
HttpHost http;
for (String s : elasticsearchNode) {
if (s.contains("http://")) {
String[] split = s.split("://");
String schema = split[0];
String ipPort = split[1];
String ip = ipPort.split(":")[0];
String port = ipPort.split(":")[1];
http = new HttpHost(ip, Integer.parseInt(port), schema);
} else {
String ip = s.split(":")[0];
String port = s.split(":")[1];
http = new HttpHost(ip, Integer.parseInt(port), "http");
}
httpHosts[index] = http;
index++;
}
return httpHosts;
}
return null;
}
}一般查询设置,针对指定索引查询
java
private SearchRequest.Builder getSearchRequest(Object[] indices, String[] types) {
if (null == indices) {
return new SearchRequest.Builder();
}
SearchRequest.Builder searchRequestBuilder = new SearchRequest.Builder();
if (indices.length > 0) {
/*筛选出指定的zipkin:span所引内容 处理当天的链路信息 索引格式：zipkin-span-2022-08-02 */
List<String> zipkinIndex = new ArrayList<>();
for (Object index : indices) {
if (((String) index).contains(Constant.SPAN)) {
zipkinIndex.add((String) index);
}
}
// 分片响应超时时间设置
searchRequestBuilder.timeout("10s");
/*集合转为数组 ----指定搜索的文档类型*/
searchRequestBuilder.index(zipkinIndex);
}
// 每个searchRequestBuilder对象build()后意味着当前对象以及结束,重新获取
return searchRequestBuilder;
}
/**
* 获取所有日志索引名称
*/
public String[] getIndices() {
String[] indices = new String[0];
try {
Request request = new Request(Constant.GET, Constant.INDICES_URL);
//发送获取所有索引名称的请求
Response response = restClient.performRequest(request);
String responseBody = EntityUtils.toString(response.getEntity());
//从响应报文中提取索引名称 除去返回值中第一个索引名称为index的索引该值为es中的默认索引名
responseBody = responseBody.substring(responseBody.indexOf('\n') + 1);
/*将所有索引名称分隔出来*/
indices = responseBody.split("\n");
} catch (IOException e) {
logger.error("获取所有日志索引名称异常：", e);
}
return indices;
}bool查询用例
java
public static List<Hit<Object>> getOriginEsTraceLinkData(Map<String, Object> typeMap, Map<String, Object> condition) {
/*临时存储数据集合*/
List<Hit<Object>> tempSearches = new ArrayList<>();
try {
LinkSearchBuilder.builder(typeMap);
long endMillis = condition.get(Constant.END_TS) == null ? 0 : (long) condition.get(Constant.END_TS);
long lookBack = condition.get(Constant.LOOK_BACK) == null ? 0 : (long) condition.get(Constant.LOOK_BACK);
long beginMillis = Math.max(endMillis - lookBack, EARLIEST_MS);
String belApp = (String) condition.get(Constant.BEL_APP);
String belLogy = (String) condition.get(Constant.BEL_LOGY);
String belSer = (String) condition.get(Constant.BEL_SER);
String traceId = (String) condition.get(Constant.TRACE_ID);
Integer limit = (Integer) condition.get(Constant.LIMIT);
long minDuration = condition.get(Constant.MIN_DURATION) == null ? 0 : (long) condition.get(Constant.MIN_DURATION);
long maxDuration = condition.get(Constant.MAX_DURATION) == null ? 0 : (long) condition.get(Constant.MAX_DURATION);
limit = (limit != null) ? limit : 10;
// bool查询
BoolQuery.Builder boolQueryBuilder = new BoolQuery.Builder();
// 根据不同的条件构建Query
if (beginMillis > 0 && endMillis > 0) {
Query rangeQueryBuilder = RangeQuery.of(builder -> builder.field(Constant.TIMESTAMP_MILLIS).gt(JsonData.of(beginMillis)).lt(JsonData.of(endMillis)))._toQuery();
boolQueryBuilder.filter(rangeQueryBuilder);
}
// tags字段下的查询不生效,tags字段对应的es模板信息不可查询
if (StrUtil.isNotEmpty(belApp)) {
Query belAppTermQueryBuilder = TermQuery.of(builder -> builder.field("belApp").value(belApp))._toQuery();
boolQueryBuilder.filter(belAppTermQueryBuilder);
}
if (StrUtil.isNotEmpty(belLogy)) {
Query belLogyTermQuery = TermQuery.of(builder -> builder.field("belLogy").value(belLogy))._toQuery();
boolQueryBuilder.filter(belLogyTermQuery);
}
if (StrUtil.isNotEmpty(belSer)) {
Query belSerTermQuery = TermQuery.of(builder -> builder.field("localEndpoint.serviceName").value(belSer))._toQuery();
boolQueryBuilder.filter(belSerTermQuery);
}
if (StrUtil.isNotEmpty(traceId)) {
Query traceIdTermQuery = TermQuery.of(builder -> builder.field("traceId").value(traceId))._toQuery();
boolQueryBuilder.filter(traceIdTermQuery);
}
if (minDuration > 0) {
Query durationQueryBuilder = RangeQuery.of(builder -> builder.field(Constant.DURATION).gt(JsonData.of(0)).lt(JsonData.of(minDuration)))._toQuery();
boolQueryBuilder.filter(durationQueryBuilder);
}
searchRequestBuilder.size(limit);
// 封装查询
searchRequestBuilder.query(boolQueryBuilder.build()._toQuery());
SearchRequest searchRequest = searchRequestBuilder.build();
System.out.println("es查询语句:" + searchRequest.toString());
// 查询结果
SearchResponse<Object> searchResponse = restHighLevelClient.search(searchRequest, Object.class);
if (searchResponse.hits().total() != null && searchResponse.hits().total().value() > 0) {
/*获取到数据*/
List<Hit<Object>> searchHits = searchResponse.hits().hits();
/*将第一次获取的数据添加到临时集合中*/
tempSearches.addAll(searchHits);
}
} catch (Exception e) {
logger.info("es数据链路信息查询异常", e);
}
return tempSearches;
}scroll查询用例
java
public static List<SpanSource> getAllData(Map<String, Object> typeMap) {
LinkSearchBuilder.builder(typeMap);
List<SpanSource> result = new ArrayList<>();
List<String> scrollIds = new ArrayList<>();
long startMillis = 0;
long endMillis = 0;
String belLogy = null;
try {
if (CollectionUtil.isNotEmpty(typeMap)) {
startMillis = typeMap.get(Constant.FROM_TIME) == null ? 0 : (long) typeMap.get(Constant.FROM_TIME);
endMillis = typeMap.get(Constant.TO_TIME) == null ? 0 : (long) typeMap.get(Constant.TO_TIME);
belLogy = (String) typeMap.get(Constant.BEL_LOGY);
}
/*每次查询分页大小设置*/
searchRequestBuilder.size(200);
// 初始化scrollId时长
searchRequestBuilder.scroll(new Time.Builder().time(SCROLL_TIME).build());
/*从数据库中获取当前创建时间*/
if ((startMillis > 0 && endMillis > 0) && (startMillis != endMillis)) {
// 多条件boolean查询
BoolQuery.Builder boolQueryBuilder = new BoolQuery.Builder();
/*范围查询 处理完成的数据的最近时间之后的所有数据进行查询该范围之后的数据*/
long finalStartMillis = startMillis;
long finalEndMillis = endMillis;
Query rangeQueryBuilder = RangeQuery.of(builder -> builder.field(Constant.TIMESTAMP_MILLIS).gt(JsonData.of(finalStartMillis)).lt(JsonData.of(finalEndMillis)))._toQuery();
boolQueryBuilder.filter(rangeQueryBuilder);
String finalBelLogy = belLogy;
Query belAppTermQueryBuilder = TermQuery.of(builder -> builder.field("belLogy").value(finalBelLogy))._toQuery();
boolQueryBuilder.filter(belAppTermQueryBuilder);
searchRequestBuilder.query(boolQueryBuilder.build()._toQuery());
} else {
// 查询所有值
searchRequestBuilder.query(QueryBuilders.matchAllQuery()._toQuery());
}
/*执行查询*/
SearchResponse<Object> searchResponse = restHighLevelClient.search(searchRequestBuilder.build(), Object.class);
if (searchResponse.hits().hits().size() > 0) {
/*临时存储数据集合*/
List<Hit<Object>> tempSearches = new ArrayList<>();
/*获取scrollId*/
String scrollId = searchResponse.scrollId();
scrollIds.add(scrollId);
/*获取到数据*/
List<Hit<Object>> searchHits = searchResponse.hits().hits();
/*将第一次获取的数据添加到临时集合中*/
tempSearches.addAll(searchHits);
/*滚动查询*/
while (searchHits.size() > 0) {
ScrollRequest.Builder builder = new ScrollRequest.Builder();
// 初始时的scrollId值
builder.scrollId(scrollId);
// scroll查询时scrollId的有效时长值必须设置
builder.scroll(new Time.Builder().time(SCROLL_TIME).build());
//响应必须是上面的响应对象，需要对上一层进行覆盖
ScrollResponse<Object> scrollResult = restHighLevelClient.scroll(builder.build(), Object.class);
scrollId = scrollResult.scrollId();
scrollIds.add(scrollId);
searchHits = scrollResult.hits().hits();
// 将数据添加到存储的集合中
tempSearches.addAll(searchHits);
}
//清除滚动
clearScrollIds(restHighLevelClient, scrollIds);
}
/***
* Description scroll滚动查询，es默认是存储500条scroll_id，
* 在一个scroll生存时间内创建超过500条继续使用滚动查询时，会报错.
* @date 2022/7/19 9:53
* @param: client
* @param: scrollIds
* @return: boolean
*/
private static boolean clearScrollIds(ElasticsearchClient client, List<String> sIds) {
//清除滚动，否则影响下次查询
ClearScrollRequest.Builder clearScrollRequest = new ClearScrollRequest.Builder();
clearScrollRequest.scrollId(sIds);
try {
client.clearScroll(clearScrollRequest.build());
return true;
} catch (IOException e) {
return false;
}
}