FastGpt知识库加载url资源

环境

一台Linux服务器，安装有：fastgpt

一台阿里云服务器，有大量pdf资源，对外通过ip访问（已开启nginx配置），姑且称为资源服务器

服务

在资源服务器提供一个接口，返回资源列表

规则：https://doc.fastgpt.cn/zh-CN/introduction/guide/knowledge_base/api_dataset

我这里采用java实现

yml

server:
  port: 8888

spring:
  application:
    name: fastgpt-file-api
  redis:
    host: 192.168.0.180
    port: 6379
    password: 123456
    database: 0

file:
  base-path: /usr/sftp/file/pdfs
  url-prefix: http://192.168.0.180/file/pdfs
  authorization: 123456
  supported-extensions: pdf,docx,md,txt,html,csv

pom

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.fastgpt</groupId>
    <artifactId>fastgpt-file-api</artifactId>
    <version>1.0.0</version>

    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>2.7.18</version>
    </parent>

    <dependencies>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-data-redis</artifactId>
        </dependency>
    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-maven-plugin</artifactId>
            </plugin>
        </plugins>
    </build>
</project>

config

package com.fastgpt.config;

import com.fasterxml.jackson.databind.ObjectMapper;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;

import javax.servlet.*;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

@Component
@Order(1)
public class AuthFilter implements Filter {

    @Value("${file.authorization}")
    private String authorization;

    private final ObjectMapper objectMapper = new ObjectMapper();

    @Override
    public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain)
            throws IOException, ServletException {
        HttpServletRequest req = (HttpServletRequest) request;
        HttpServletResponse resp = (HttpServletResponse) response;

        if (req.getRequestURI().startsWith("/fastgpt/v1/file/")) {
            String header = req.getHeader("Authorization");
            String expected = "Bearer " + authorization;
            if (header == null || !header.equals(expected)) {
                resp.setStatus(401);
                resp.setContentType("application/json;charset=UTF-8");
                Map<String, Object> body = new HashMap<>();
                body.put("code", 401);
                body.put("success", false);
                body.put("message", "Unauthorized");
                resp.getWriter().write(objectMapper.writeValueAsString(body));
                return;
            }
        }
        chain.doFilter(request, response);
    }
}


package com.fastgpt.config;


import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.serializer.GenericJackson2JsonRedisSerializer;
import org.springframework.data.redis.serializer.StringRedisSerializer;

@Configuration
public class RedisConfig {
    @Bean
    public RedisTemplate<String, Object> redisTemplate(RedisConnectionFactory factory) {
        RedisTemplate<String, Object> template = new RedisTemplate<>();
        template.setConnectionFactory(factory);
        template.setKeySerializer(new StringRedisSerializer());
        template.setValueSerializer(new GenericJackson2JsonRedisSerializer());
        template.setHashKeySerializer(new StringRedisSerializer());
        template.setHashValueSerializer(new GenericJackson2JsonRedisSerializer());
        template.afterPropertiesSet();
        return template;
    }
}

实体

package com.fastgpt.dto;

import com.fasterxml.jackson.annotation.JsonInclude;

@JsonInclude(JsonInclude.Include.NON_NULL)
public class ApiResponse<T> {
    private int code = 200;
    private boolean success = true;
    private String message = "";
    private T data;

    public static <T> ApiResponse<T> ok(T data) {
        ApiResponse<T> r = new ApiResponse<>();
        r.data = data;
        return r;
    }

    public static <T> ApiResponse<T> fail(String message) {
        ApiResponse<T> r = new ApiResponse<>();
        r.code = 400;
        r.success = false;
        r.message = message;
        return r;
    }

    public int getCode() { return code; }
    public void setCode(int code) { this.code = code; }
    public boolean isSuccess() { return success; }
    public void setSuccess(boolean success) { this.success = success; }
    public String getMessage() { return message; }
    public void setMessage(String message) { this.message = message; }
    public T getData() { return data; }
    public void setData(T data) { this.data = data; }
}

package com.fastgpt.dto;

import com.fasterxml.jackson.annotation.JsonInclude;

import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;

@JsonInclude(JsonInclude.Include.NON_NULL)
public class FileItemDTO {
    private String id;
    private String name;
    private String parentId;
    private String type;   // "file" or "folder"
    private String createTime;
    private String updateTime;
    private Boolean hasChild;

    private static final DateTimeFormatter ISO_FMT =
            DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")
                    .withZone(ZoneId.of("UTC"));

    private static String fmt(long millis) {
        return ISO_FMT.format(Instant.ofEpochMilli(millis));
    }

    public static FileItemDTO folder(String id, String name, String parentId, long updateTime, boolean hasChild) {
        FileItemDTO dto = new FileItemDTO();
        dto.id = id;
        dto.name = name;
        dto.parentId = parentId;
        dto.type = "folder";
        dto.createTime = fmt(updateTime);
        dto.updateTime = fmt(updateTime);
        dto.hasChild = hasChild;
        return dto;
    }

    public static FileItemDTO file(String id, String name, String parentId, long updateTime) {
        FileItemDTO dto = new FileItemDTO();
        dto.id = id;
        dto.name = name;
        dto.parentId = parentId;
        dto.type = "file";
        dto.createTime = fmt(updateTime);
        dto.updateTime = fmt(updateTime);
        dto.hasChild = false;
        return dto;
    }

    public String getId() { return id; }
    public void setId(String id) { this.id = id; }
    public String getName() { return name; }
    public void setName(String name) { this.name = name; }
    public String getParentId() { return parentId; }
    public void setParentId(String parentId) { this.parentId = parentId; }
    public String getType() { return type; }
    public void setType(String type) { this.type = type; }
    public String getCreateTime() { return createTime; }
    public void setCreateTime(String createTime) { this.createTime = createTime; }
    public String getUpdateTime() { return updateTime; }
    public void setUpdateTime(String updateTime) { this.updateTime = updateTime; }
    public Boolean getHasChild() { return hasChild; }
    public void setHasChild(Boolean hasChild) { this.hasChild = hasChild; }
}

ServiceImpl 业务实现

package com.fastgpt.service;

import com.fastgpt.dto.FileItemDTO;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;

import javax.annotation.PostConstruct;
import com.fasterxml.jackson.databind.ObjectMapper;

import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.*;
import java.util.stream.Collectors;

@Service
public class FileService {

    @Value("${file.base-path}")
    private String basePath;

    @Value("${file.url-prefix}")
    private String urlPrefix;

    @Value("${file.supported-extensions}")
    private String supportedExtensions;

    private final RedisTemplate<String, Object> redisTemplate;

    private static final String KEY_LIST = "fastgpt:files:list";
    private static final String KEY_TIME = "fastgpt:files:lastTime";

    private static final Set<String> TEXT_EXTENSIONS = new HashSet<>(Arrays.asList("txt", "md", "html", "csv"));

    public FileService(RedisTemplate<String, Object> redisTemplate) {
        this.redisTemplate = redisTemplate;
    }

    private Set<String> getSupportedExtSet() {
        return Arrays.stream(supportedExtensions.split(","))
                .map(String::trim)
                .map(String::toLowerCase)
                .collect(Collectors.toSet());
    }

    @PostConstruct
    public void init() {
        List<FileItemDTO> cache = getCachedList();
        if (cache == null || cache.isEmpty()) {
            fullScan();
        }
    }

    @Scheduled(fixedRate = 600000)
    public void update() {
        Long last = (Long) redisTemplate.opsForValue().get(KEY_TIME);
        if (last == null) {
            fullScan();
        } else {
            fullScan();
        }
    }

    // ---- public API ----

    public List<FileItemDTO> listFiles(String parentId, String searchKey) {
        List<FileItemDTO> all = getCachedList();
        if (all == null) {
            fullScan();
            all = getCachedList();
        }
        if (all == null) all = new ArrayList<>();

        if (parentId == null) parentId = "";
        final String pid = parentId;

        List<FileItemDTO> result = all.stream()
                .filter(f -> pid.equals(f.getParentId() == null ? "" : f.getParentId()))
                .collect(Collectors.toList());

        if (StringUtils.hasText(searchKey)) {
            String key = searchKey.toLowerCase();
            result = result.stream()
                    .filter(f -> f.getName().toLowerCase().contains(key))
                    .collect(Collectors.toList());
        }

        return result;
    }

    public String getFileContent(String id) {
        File file = resolveFile(id);
        if (file == null || !file.exists() || file.isDirectory()) return null;

        String ext = getExtension(file.getName());
        if (TEXT_EXTENSIONS.contains(ext)) {
            try {
                return new String(Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8);
            } catch (IOException e) {
                return null;
            }
        }
        return null;
    }

    public String getPreviewUrl(String id) {
        File file = resolveFile(id);
        if (file == null || !file.exists()) return null;
        return buildUrl(id);
    }

    public FileItemDTO getFileDetail(String id) {
        if (id == null || id.isEmpty()) return null;

        // prefer cached data (has hasChild computed)
        List<FileItemDTO> all = getCachedList();
        if (all != null) {
            for (FileItemDTO item : all) {
                if (id.equals(item.getId())) return item;
            }
        }

        // fallback: stat the file directly
        File file = resolveFile(id);
        if (file == null || !file.exists()) return null;

        if (file.isDirectory()) {
            boolean hasChild = file.list() != null && file.list().length > 0;
            return FileItemDTO.folder(id, file.getName(), getParentId(id), file.lastModified(), hasChild);
        } else {
            return FileItemDTO.file(id, file.getName(), getParentId(id), file.lastModified());
        }
    }

    // ---- scan logic ----

    synchronized void fullScan() {
        List<FileItemDTO> list = new ArrayList<>();
        scanDir(new File(basePath), "", list);

        // second pass: compute hasChild for folders
        Set<String> parentIds = new HashSet<>();
        for (FileItemDTO item : list) {
            String pid = item.getParentId();
            if (pid != null && !pid.isEmpty()) {
                parentIds.add(pid);
            }
        }
        for (FileItemDTO item : list) {
            if ("folder".equals(item.getType())) {
                item.setHasChild(parentIds.contains(item.getId()));
            }
        }

        redisTemplate.opsForValue().set(KEY_LIST, list);
        redisTemplate.opsForValue().set(KEY_TIME, System.currentTimeMillis());
    }

    private void scanDir(File dir, String relativePath, List<FileItemDTO> list) {
        if (dir == null || !dir.exists() || !dir.isDirectory()) return;
        File[] files = dir.listFiles();
        if (files == null) return;

        Set<String> extSet = getSupportedExtSet();

        for (File f : files) {
            if (f.isDirectory()) {
                String childPath = relativePath.isEmpty() ? f.getName() : relativePath + "/" + f.getName();
                // hasChild computed in second pass; placeholder false for now
                list.add(FileItemDTO.folder(childPath, f.getName(),
                        relativePath.isEmpty() ? "" : relativePath, f.lastModified(), false));
                scanDir(f, childPath, list);
            } else {
                String ext = getExtension(f.getName());
                if (!extSet.contains(ext)) continue;

                String id = relativePath.isEmpty() ? f.getName() : relativePath + "/" + f.getName();
                String parentId = relativePath.isEmpty() ? "" : relativePath;
                list.add(FileItemDTO.file(id, f.getName(), parentId, f.lastModified()));
            }
        }
    }

    // ---- helpers ----

    private File resolveFile(String id) {
        if (id == null || id.isEmpty()) return null;
        return new File(basePath, id);
    }

    private String buildUrl(String id) {
        return urlPrefix + "/" + id;
    }

    private String getParentId(String id) {
        if (id == null || id.isEmpty()) return "";
        int idx = id.lastIndexOf('/');
        if (idx < 0) return "";
        return id.substring(0, idx);
    }

    private String getExtension(String name) {
        int idx = name.lastIndexOf('.');
        if (idx < 0) return "";
        return name.substring(idx + 1).toLowerCase();
    }

    // ---- cache ----

    private final ObjectMapper objectMapper = new ObjectMapper();

    @SuppressWarnings("unchecked")
    private List<FileItemDTO> getCachedList() {
        try {
            Object val = redisTemplate.opsForValue().get(KEY_LIST);
            if (val instanceof List) {
                List<?> raw = (List<?>) val;
                if (raw.isEmpty()) return new ArrayList<>();
                if (raw.get(0) instanceof FileItemDTO) return (List<FileItemDTO>) raw;
                List<FileItemDTO> converted = new ArrayList<>();
                for (Object item : raw) {
                    converted.add(objectMapper.convertValue(item, FileItemDTO.class));
                }
                return converted;
            }
        } catch (Exception e) {
            // stale/missing cache --- rebuild
        }
        return null;
    }
}

Controller

package com.fastgpt.controller;

import com.fastgpt.dto.ApiResponse;
import com.fastgpt.dto.FileItemDTO;
import com.fastgpt.service.FileService;
import org.springframework.web.bind.annotation.*;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

@RestController
@RequestMapping("/fastgpt")
public class FileController {

    private final FileService fileService;

    public FileController(FileService fileService) {
        this.fileService = fileService;
    }

    // ---- 文件树 ----

    @PostMapping("/v1/file/list")
    public ApiResponse<List<FileItemDTO>> list(@RequestBody(required = false) Map<String, Object> body) {
        String parentId = body != null ? str(body.get("parentId")) : "";
        String searchKey = body != null ? str(body.get("searchKey")) : "";
        return ApiResponse.ok(fileService.listFiles(parentId, searchKey));
    }

    // ---- 文件内容 ----

    @GetMapping("/v1/file/content")
    public ApiResponse<Map<String, Object>> contentGet(@RequestParam("id") String id) {
        return buildContent(id);
    }

    @PostMapping("/v1/file/content")
    public ApiResponse<Map<String, Object>> contentPost(@RequestBody(required = false) Map<String, Object> body) {
        return buildContent(body != null ? str(body.get("id")) : "");
    }

    private ApiResponse<Map<String, Object>> buildContent(String id) {
        if (id.isEmpty()) return ApiResponse.fail("id is required");

        FileItemDTO detail = fileService.getFileDetail(id);
        if (detail == null) return ApiResponse.fail("file not found");

        Map<String, Object> data = new HashMap<>();
        data.put("title", detail.getName());
        String content = fileService.getFileContent(id);
        if (content != null) {
            data.put("content", content);
        } else {
            data.put("previewUrl", fileService.getPreviewUrl(id));
        }
        return ApiResponse.ok(data);
    }

    // ---- 文件详情 ----

    @GetMapping("/v1/file/detail")
    public ApiResponse<FileItemDTO> detailGet(@RequestParam("id") String id) {
        return buildDetail(id);
    }

    @PostMapping("/v1/file/detail")
    public ApiResponse<FileItemDTO> detailPost(@RequestBody(required = false) Map<String, Object> body) {
        return buildDetail(body != null ? str(body.get("id")) : "");
    }

    private ApiResponse<FileItemDTO> buildDetail(String id) {
        if (id.isEmpty()) return ApiResponse.fail("id is required");
        FileItemDTO detail = fileService.getFileDetail(id);
        if (detail == null) return ApiResponse.fail("file not found");
        return ApiResponse.ok(detail);
    }

    // ---- 文件阅读链接 ----

    @GetMapping("/v1/file/read")
    public ApiResponse<Map<String, String>> readGet(@RequestParam("id") String id) {
        return buildRead(id);
    }

    @PostMapping("/v1/file/read")
    public ApiResponse<Map<String, String>> readPost(@RequestBody(required = false) Map<String, Object> body) {
        return buildRead(body != null ? str(body.get("id")) : "");
    }

    private ApiResponse<Map<String, String>> buildRead(String id) {
        if (id.isEmpty()) return ApiResponse.fail("id is required");
        String url = fileService.getPreviewUrl(id);
        if (url == null) return ApiResponse.fail("file not found");
        Map<String, String> data = new HashMap<>();
        data.put("url", url);
        return ApiResponse.ok(data);
    }

    private static String str(Object val) {
        return val != null ? val.toString() : "";
    }
}

FastGpt知识库配置

这里配置的是：http://192.168.0.180:8888/fastgpt

接口在请求的时候会自动拼，前缀我们给定就行