日志分析删除

日志分析

场景

运维嫌弃生产环境打印日志过多，而且日志存储需要费用，让我们减少打印日志大小，所以需要分析日志在哪里打印的过多

解决方案

读取生产日志文件，统计分析打印日志的地方，最后删除代码中打印日志的地方

实现-LogParAnalyzer

public class LogParAnalyzer {


    //日志原始文件
    private File log;
    private List<Pattern> list;
    private ExecutorService executorService;

    //生成的分割文件
    private String subPath = "D:\\split\\";
    private List<File> files;

    public LogParAnalyzer(File log, List<String> patterns) {
        this.log = log;
        executorService = Executors.newFixedThreadPool(30);
        list = new ArrayList<>();
        try {
            for (String pattern : patterns) {
                Pattern p = Pattern.compile(pattern);
                list.add(p);
            }
        } catch (Exception e) {
            throw new RuntimeException(e);
        }

    }



    public void analyze() throws Exception {
        // 使用 try-with-resources 自动关闭 BufferedReader
        int chunkSize = 100000;
        try (BufferedReader reader = new BufferedReader(new FileReader(log))) {
            File file = new File(subPath);
            if (!file.exists()) {
                file.mkdirs();
            }
            String line;
            List<CompletableFuture<?>> task = new ArrayList<>();
            int cur = 0;
            List<String> list = new ArrayList<>();
            AtomicInteger batch = new AtomicInteger(0);
            while ((line = reader.readLine()) != null) {
                //sb 会通过Arrays.copy复制字节数组，内存频繁复制
                list.add(line);
                cur++;
                if ((cur % chunkSize) == 0) {
                    //深拷贝
                    List<String> tt = list.stream().map(String::new).collect(Collectors.toList());
                    list.clear();
                    CompletableFuture f =
                            CompletableFuture.runAsync(() -> processChunk(tt, batch.get()), executorService);
                    task.add(f);
                    batch.incrementAndGet();
                }
            }
            if (list.size()>0) {
                CompletableFuture f =
                        CompletableFuture.runAsync(() -> processChunk(list, batch.get()), executorService);
                task.add(f);
            }
            //等待所有任务结束
            CompletableFuture.allOf(task.toArray(new CompletableFuture[0])).get();
            System.out.println("task execute finished");
        }
    }


    private void processChunk(List<String> lines, int batch) {
        try {
            System.out.println(Thread.currentThread().getName()+" execute "+ batch+".txt start");
            Map<String, AtomicInteger> map = new HashMap<>();
            try (BufferedWriter writer = Files.newBufferedWriter(Paths.get(subPath + batch + ".txt"))) {
                lines.forEach(line -> {
                    for (Pattern pattern : list) {
                        Matcher matcher = pattern.matcher(line);
                        if (matcher.find()) {
                            String group = matcher.group(0);
                            map.computeIfAbsent(group, k -> new AtomicInteger(1)).incrementAndGet();
                        }
                    }

                    if (map.size() > 0) {
                        //每个文件只保存100前100条
                        writeBatchToFile(writer, map);
                    }
                });
            }
            System.out.println(Thread.currentThread().getName()+" execute "+ batch+".txt end");
        } catch (Exception e) {
            e.printStackTrace();
        }

    }

    private void writeBatchToFile(BufferedWriter writer, Map<String, AtomicInteger> map) {
        Map<String, AtomicInteger> limit = limit(map, 100);
        try {
            for (Map.Entry<String, AtomicInteger> entry : limit.entrySet()) {
                writer.write(entry.getKey() + "=" + entry.getValue().get());
                writer.newLine();
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
        //清除缓存
        map.clear();
        limit.clear();
    }

    public void mergeAndSort() throws Exception {
        files = Files.list(Paths.get(subPath))
                .map(Path::toFile)
                .filter(f -> f.length() > 0)
                .collect(Collectors.toList());

        // 创建 ForkJoinPool
        ForkJoinPool forkJoinPool = new ForkJoinPool();
        MergeFileTask mergeFileTask = new MergeFileTask(files.toArray(new File[0]), forkJoinPool);
        Path finalPath = mergeFileTask.invoke();
        System.out.println("final path: " + finalPath.toAbsolutePath());
        try (BufferedReader reader = Files.newBufferedReader(finalPath)) {
            String line;
            while ((line = reader.readLine()) != null) {
                System.out.println(line);
            }
        }
        mergeFileTask.finished();
    }

    public void finished() throws IOException {
        if (!CollectionUtils.isEmpty(files)){
            files.stream().parallel().forEach(File::delete);
        }
        Files.deleteIfExists(Paths.get(subPath));
    }

    public Map<String, AtomicInteger> limit(Map<String, AtomicInteger> map, int limit) {
        // 排序并过滤结果
        return map.entrySet().stream()
                .sorted(Map.Entry.comparingByValue(Comparator.comparing(AtomicInteger::get).reversed()))
                .limit(limit)
                .collect(Collectors.toMap(
                        Map.Entry::getKey,
                        Map.Entry::getValue,
                        (oldValue, newValue) -> oldValue, // 解决键冲突
                        LinkedHashMap::new
                ));
    }

}

测试

@Test
public void bb() throws Exception {
    StopWatch stopWatch = new StopWatch();
    stopWatch.start();
    File log = new File("E:\\log_total.log\\log_total.log");
    //2023-09-26 11:10:00.123 INFO - none --- [main] com.example.service.UserService.create:42 - User service started successfully.
    //匹配出 com.example.service.UserService.create:42
    //c.y.c.w..*指文件名太长，缩写
    List<String> list = Arrays.asList("(com\\.xx\\.[\\w\\.\\*]*:\\d*)", "(c\\.y\\.c\\.[\\w\\.\\*]*:\\d*)");
    LogParAnalyzer logAnalyzer = new LogParAnalyzer(log, list);
    logAnalyzer.analyze();
    logAnalyzer.mergeAndSort();
    logAnalyzer.finished();
    stopWatch.stop();
    System.out.println(stopWatch.prettyPrint());
    //最后统计格式
    //c.y.c.s.service.impl.Service.complete:98 count: 6
}

good luck!