日志分析
场景
运维嫌弃生产环境打印日志过多，而且日志存储需要费用，让我们减少打印日志大小，所以需要分析日志在哪里打印的过多
解决方案
读取生产日志文件，统计分析打印日志的地方，最后删除代码中打印日志的地方
实现-LogParAnalyzer
java
public class LogParAnalyzer {
//日志原始文件
private File log;
private List<Pattern> list;
private ExecutorService executorService;
//生成的分割文件
private String subPath = "D:\\split\\";
private List<File> files;
public LogParAnalyzer(File log, List<String> patterns) {
this.log = log;
executorService = Executors.newFixedThreadPool(30);
list = new ArrayList<>();
try {
for (String pattern : patterns) {
Pattern p = Pattern.compile(pattern);
list.add(p);
}
} catch (Exception e) {
throw new RuntimeException(e);
}
}
public void analyze() throws Exception {
// 使用 try-with-resources 自动关闭 BufferedReader
int chunkSize = 100000;
try (BufferedReader reader = new BufferedReader(new FileReader(log))) {
File file = new File(subPath);
if (!file.exists()) {
file.mkdirs();
}
String line;
List<CompletableFuture<?>> task = new ArrayList<>();
int cur = 0;
List<String> list = new ArrayList<>();
AtomicInteger batch = new AtomicInteger(0);
while ((line = reader.readLine()) != null) {
//sb 会通过Arrays.copy复制字节数组，内存频繁复制
list.add(line);
cur++;
if ((cur % chunkSize) == 0) {
//深拷贝
List<String> tt = list.stream().map(String::new).collect(Collectors.toList());
list.clear();
CompletableFuture f =
CompletableFuture.runAsync(() -> processChunk(tt, batch.get()), executorService);
task.add(f);
batch.incrementAndGet();
}
}
if (list.size()>0) {
CompletableFuture f =
CompletableFuture.runAsync(() -> processChunk(list, batch.get()), executorService);
task.add(f);
}
//等待所有任务结束
CompletableFuture.allOf(task.toArray(new CompletableFuture[0])).get();
System.out.println("task execute finished");
}
}
private void processChunk(List<String> lines, int batch) {
try {
System.out.println(Thread.currentThread().getName()+" execute "+ batch+".txt start");
Map<String, AtomicInteger> map = new HashMap<>();
try (BufferedWriter writer = Files.newBufferedWriter(Paths.get(subPath + batch + ".txt"))) {
lines.forEach(line -> {
for (Pattern pattern : list) {
Matcher matcher = pattern.matcher(line);
if (matcher.find()) {
String group = matcher.group(0);
map.computeIfAbsent(group, k -> new AtomicInteger(1)).incrementAndGet();
}
}
if (map.size() > 0) {
//每个文件只保存100前100条
writeBatchToFile(writer, map);
}
});
}
System.out.println(Thread.currentThread().getName()+" execute "+ batch+".txt end");
} catch (Exception e) {
e.printStackTrace();
}
}
private void writeBatchToFile(BufferedWriter writer, Map<String, AtomicInteger> map) {
Map<String, AtomicInteger> limit = limit(map, 100);
try {
for (Map.Entry<String, AtomicInteger> entry : limit.entrySet()) {
writer.write(entry.getKey() + "=" + entry.getValue().get());
writer.newLine();
}
} catch (IOException e) {
e.printStackTrace();
}
//清除缓存
map.clear();
limit.clear();
}
public void mergeAndSort() throws Exception {
files = Files.list(Paths.get(subPath))
.map(Path::toFile)
.filter(f -> f.length() > 0)
.collect(Collectors.toList());
// 创建 ForkJoinPool
ForkJoinPool forkJoinPool = new ForkJoinPool();
MergeFileTask mergeFileTask = new MergeFileTask(files.toArray(new File[0]), forkJoinPool);
Path finalPath = mergeFileTask.invoke();
System.out.println("final path: " + finalPath.toAbsolutePath());
try (BufferedReader reader = Files.newBufferedReader(finalPath)) {
String line;
while ((line = reader.readLine()) != null) {
System.out.println(line);
}
}
mergeFileTask.finished();
}
public void finished() throws IOException {
if (!CollectionUtils.isEmpty(files)){
files.stream().parallel().forEach(File::delete);
}
Files.deleteIfExists(Paths.get(subPath));
}
public Map<String, AtomicInteger> limit(Map<String, AtomicInteger> map, int limit) {
// 排序并过滤结果
return map.entrySet().stream()
.sorted(Map.Entry.comparingByValue(Comparator.comparing(AtomicInteger::get).reversed()))
.limit(limit)
.collect(Collectors.toMap(
Map.Entry::getKey,
Map.Entry::getValue,
(oldValue, newValue) -> oldValue, // 解决键冲突
LinkedHashMap::new
));
}
}测试
bash
@Test
public void bb() throws Exception {
StopWatch stopWatch = new StopWatch();
stopWatch.start();
File log = new File("E:\\log_total.log\\log_total.log");
//2023-09-26 11:10:00.123 INFO - none --- [main] com.example.service.UserService.create:42 - User service started successfully.
//匹配出 com.example.service.UserService.create:42
//c.y.c.w..*指文件名太长，缩写
List<String> list = Arrays.asList("(com\\.xx\\.[\\w\\.\\*]*:\\d*)", "(c\\.y\\.c\\.[\\w\\.\\*]*:\\d*)");
LogParAnalyzer logAnalyzer = new LogParAnalyzer(log, list);
logAnalyzer.analyze();
logAnalyzer.mergeAndSort();
logAnalyzer.finished();
stopWatch.stop();
System.out.println(stopWatch.prettyPrint());
//最后统计格式
//c.y.c.s.service.impl.Service.complete:98 count: 6
}good luck!