基于数据库的全文检索实现

weixin_448771722024-03-11 15:02

对于内容摘要，信件内容进行全文检索
基于SpringBoot 2.5.6+Postgresql+jpa+hibernate实现

依赖

xml 复制代码 
<spring-boot.version>2.5.6</spring-boot.version>
<hibernate-types-52.version>2.14.0</hibernate-types-52.version>

<dependency>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-data-jpa</artifactId>
</dependency>
<!-- hibernate支持配置 -->
<dependency>
    <groupId>org.hibernate</groupId>
    <artifactId>hibernate-core</artifactId>
</dependency>
<dependency>
    <groupId>org.hibernate</groupId>
    <artifactId>hibernate-entitymanager</artifactId>
</dependency>
<dependency>
    <groupId>org.hibernate</groupId>
    <artifactId>hibernate-ehcache</artifactId>
</dependency>
<dependency>
    <groupId>com.vladmihalcea</groupId>
    <artifactId>hibernate-types-52</artifactId>
    <version>${hibernate-types-52.version}</version>
</dependency>
<!-- hibernate支持配置 -->
 
<dependency>
    <groupId>org.postgresql</groupId>
    <artifactId>postgresql</artifactId>
</dependency>

<dependencyManagement>
    <dependencies>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-parent</artifactId>
            <version>${spring-boot.version}</version>
            <type>pom</type>
            <scope>import</scope>
        </dependency>
    </dependencies>
</dependencyManagement>

业务逻辑

登记保存之后，处理完成业务逻辑，发送全文检索事件

java 复制代码 
//附加类型对应的附件ids
Map<String, List<String>> attCategoryToAttIds = new HashMap<String, List<String>>();
attCategoryToAttIds.put(cmpRecord.getFileCategory(), files==null?null:files.stream().map(d->d.getId()).collect(Collectors.toList()));
//处理监听事件所需要的数据
Map<String,Object>eventData = Utils.buildMap("recordId", cmpRecord.getId(),"newRecord", true,"attCategoryToAttIds", attCategoryToAttIds);
//创建全文检索事件
DomainEvent de = new DefaultDomainEvent(cmpRecord.getId() + "_Handle_CmpRecord_FullTextSearch", operateInfo, ExecutePoint.CURR_THREAD,
                eventData, new Date(), "Handle_CmpRecord_FullTextSearch");
//发布事件
DomainEventPublisherFactory.getRegisteredPublisher().publishEvent(de);

处理业务发送全文检索事件

java 复制代码 
@Service
@Transactional
@SuppressWarnings("unchecked")
public class HandleCmpRecordFullTextSearchListener implements IDomainEventListener {
    
    @Autowired
    private CmpRecordRepository cmpRecordRepository;
    @Autowired
    private DataChangeLogEventRepository dataChangeLogEventRepository;
    
    @Override
    public void onEvent(DomainEvent event) {
        AccessTokenUser operator=event.getOperator();
        Date operateTime=event.obtainEventTime();
        Map<String,Object> otherData=(Map<String,Object>)event.getEventData();
        String recordId = (String) otherData.get("recordId");
        boolean newRecord=(boolean)otherData.get("newRecord");
        String comment = (String) otherData.get("comment");//办理记录的备注
        if(StringUtils.isBlank(recordId)) {
            throw new RuntimeException("未指定信访记录id");
        }
     	//获取登记信息
        CmpRecord cmdRecord = cmpRecordRepository.getCmpRecordById(recordId);
        //指定关联关系
        RelateProjValObj cmpRdProj=new RelateProjValObj(recordId,RelateProjConstants.PROJTYPE_CMP_RECORD); 
        //这是关联那个业务
        List<RelateProjValObj> mainProjs=Arrays.asList(cmpRdProj);
        DomainEvent de=null;
        //登记信息是无效的 则删除已存在的和这个件相关的
        if(cmdRecord==null||!cmdRecord.isValidEntity()) {
            //删除全文检索信息
            de=new FullTextSearchOperateEvent(recordId+"_FullTextSearch_Remove", null, operator, operateTime,
                    mainProjs, null);
            DomainEventPublisherFactory.getRegisteredPublisher().publishEvent(de);
            return;
        }
        //全文检索 类型前缀 
        String contentTypepPefix=RelateProjConstants.PROJTYPE_CMP_RECORD;
        //在当前线程中执行，保证事务一致性
        ExecutePoint executePoint=ExecutePoint.CURR_THREAD;
        
        /***********************************************关键词检索-内容摘要***********************************************/
        //全文检索的类型 区分内容摘要 附件内容
        List<String> contentTypes=Arrays.asList(contentTypepPefix+"_contentAbstract");
        String contentAbstract =cmdRecord.getBaseInfo().getContentAbstract();//内容摘要
        if(StringUtils.isBlank(contentAbstract)) contentAbstract="";
        if(StringUtils.isNotBlank(comment)) {
            if(StringUtils.isNotBlank(contentAbstract)) contentAbstract=contentAbstract + ",";
            contentAbstract=contentAbstract+comment;
        }
        de=new FullTextSearchOperateEvent(recordId+"_FullTextSearch_Update", executePoint, operator, operateTime,
                mainProjs, contentTypes, contentAbstract, null);
        DomainEventPublisherFactory.getRegisteredPublisher().publishEvent(de);
        
        /***********************************************关键词检索-信件内容***********************************************/
        contentTypes=Arrays.asList(contentTypepPefix+"_content");
        String content =cmdRecord.getBaseInfo().getContent();//信件内容
        de=new FullTextSearchOperateEvent(recordId+"_FullTextSearch_Update", executePoint, operator, operateTime,
                mainProjs, contentTypes, content, null);
        DomainEventPublisherFactory.getRegisteredPublisher().publishEvent(de);
        
        /***********************************************关键词检索-附件(原信等)***********************************************/
        //如果附件也需要检索  设置attIds参数
        Map<String,List<String>> attCategoryToAttIds=(Map<String,List<String>>)otherData.get("attCategoryToAttIds");
        if(attCategoryToAttIds!=null && attCategoryToAttIds.size() > 0) {
            //按附件类型分开
            for (Map.Entry<String,List<String>> d : attCategoryToAttIds.entrySet()) {
                contentTypes=Arrays.asList(contentTypepPefix+"_att_"+d.getKey());
                List<String> attIds=d.getValue();//公文相关附件
                de=new FullTextSearchOperateEvent(recordId+"_att_"+d.getKey()+"_FullTextSearch_Update", executePoint,
                        operator, operateTime, mainProjs, contentTypes, null, attIds);
                DomainEventPublisherFactory.getRegisteredPublisher().publishEvent(de);
            }
        }
    }
    
    @Override
    public boolean listenOn(String eventType) {
        return "Handle_CmpRecord_FullTextSearch".equals(eventType);
    }
}

统一处理全文检索事件

java 复制代码 
@Service
@Transactional
public class FullTextSearchListener extends JpaHibernateRepository implements IDomainEventListener{
    
	@Autowired
	private FullTextSearchRepository fullTextSearchRepository;
	@Autowired
	private IFileSysService fileSysService;
	
    @Override
    public void onEvent(DomainEvent event) {
    	if("true".equals(BaseConstants.getProperty("prefetchingRecordNo", "false"))){
    		return;
    	}
        FullTextSearchOperateEvent de = null;
        if(event instanceof FullTextSearchOperateEvent) {
        	de=(FullTextSearchOperateEvent)event;
        }
        if(de==null) {
        	return;
        }
        if(FullTextSearchOperateEvent.EVENTTYPE_UPDATE.equals(de.getEventType())) {
        	/**
        	 "mainProjs":List<RelateProjValObj> 必选
        	 "contentType":String 必选
        	 "content":String 可选
        	 "attIds":List<String> 可选  content与attIds都不存在 会删除对应关键词检索
        	 "relProjs":List<RelateProjValObj> 可选  指定的需要添加的关系
        	 "removeOtherRelProjs":false  可选  是否清除 指定relProjs以外的关联记录
        	 */
        	this.fullTextSearchUpdate(de);
        }else if(FullTextSearchOperateEvent.EVENTTYPE_REMOVE.equals(de.getEventType())) {
        	/**
        	 "mainProjs":List<RelateProjValObj> 必选
        	 */
        	this.fullTextSearchRemoveByProjs(de);
        }
    }
    
	//关键词检索增加
	private void fullTextSearchUpdate(FullTextSearchOperateEvent de) {
		Date date=de.obtainEventTime();
		if(date==null) {
			date=new Date();
		}
		List<RelateProjValObj> mainProjs=de.getMainProjs();
		String contentType=null;
		if(de.getContentTypes()!=null&&de.getContentTypes().size()==1) {
			contentType=de.getContentTypes().get(0);
		}
		String content=de.getContent();
		List<String> attIds=de.getAttIds();
		if(mainProjs==null||mainProjs.size()==0
				||StringUtils.isBlank(contentType)
				) {
			throw new RuntimeException("数据指定错误");
		}
		Set<String> fullTextIds=new HashSet<String>();
		for (RelateProjValObj mainProj : mainProjs) {
			if(StringUtils.isBlank(mainProj.getProjId())||StringUtils.isBlank(mainProj.getProjType())) {
				continue;
			}
			fullTextIds.add(new FullTextSearch(mainProj,contentType,null,null).getId());
		}
		if(fullTextIds.size()==0) {
			throw new RuntimeException("数据指定错误");
		}
		//这是从附件中获取文本数据
		if(StringUtils.isBlank(content)&&attIds!=null) {
			content="";
			try {
				if(attIds.size()>0) {
					Map<String,String> attIdToContentMao=ThreadLocalCache.fetchAPIData(null,()->{
						return fileSysService.findFileContentByIds(attIds, true);
					});
					for (String attContent : attIdToContentMao.values()) {
						if(StringUtils.isBlank(attContent)) {
							continue;
						}
						if(StringUtils.isNotBlank(content)) {
							content+=",";
						}
						content+=RegExUtils.replaceAll(attContent, "\\u0000", "");//处理掉非法字符
					}
				}
			} catch (Exception e) {
				e.printStackTrace();
			}
		}
		//从数据库中获取已经存的
		List<FullTextSearch> oldFullTexts=this.fullTextSearchRepository.findFullTextSearchByIds(fullTextIds);
		Map<String,FullTextSearch> oldFullTextMap=oldFullTexts.stream().collect(Collectors.toMap(d->d.getId(),d->d));
		//遍历这次需要更新的记录
		for (RelateProjValObj mainProj : mainProjs) {
			if(StringUtils.isBlank(mainProj.getProjId())||StringUtils.isBlank(mainProj.getProjType())) {
				continue;
			}
			FullTextSearch fullText=new FullTextSearch(mainProj, contentType, content, date);
			
			FullTextSearch oldFullText=oldFullTextMap.get(fullText.getId());
			//旧的记录中已存在 则更新
			if(oldFullText!=null) {
				if(StringUtils.isBlank(content)) {
				//如果内容未空 则删除	
				this.fullTextSearchRepository.removeFullTextSearch(oldFullText);
					return;
				}
				//如果存在内容，则更新
				this.fullTextSearchRepository
				.updateFullTextSearchContent(fullText.getId(), content, date);
			}else {
				if(StringUtils.isBlank(content)) {
					return;
				}
				try {//否则 创建全文检索记录
					this.fullTextSearchRepository.createFullTextSearch(fullText);
				} catch (Exception e) {
					e.printStackTrace();
					return;
				}
			}
			
		}
	}
	
	//关键词检索删除  根据主相关件
	private void fullTextSearchRemoveByProjs(FullTextSearchOperateEvent de) {
		Date date=de.obtainEventTime();
		if(date==null) {
			date=new Date();
		}
		List<RelateProjValObj> mainProjs=de.getMainProjs();
		if(mainProjs==null||mainProjs.size()==0) {
			throw new RuntimeException("数据指定错误");
		}
		
		List<String> projKeys=new ArrayList<String>();
		for (RelateProjValObj mainProj : mainProjs) {
			projKeys.add(mainProj.getProjKey());
		}
		Map<String,Object> params=new HashMap<String,Object>();
		StringBuffer hql=new StringBuffer();
		hql.append("delete from ").append(FullTextSearch.class.getName()).append(" ");
		hql.append("where mainProj.projKey IN(:projKeys) ");
		params.put("projKeys", projKeys);
		if(de.getContentTypes()!=null&&de.getContentTypes().size()>0) {
			params.put("contentTypes", de.getContentTypes());
		}
		this.createHQLQueryByMapParams(hql.toString(), params).executeUpdate();
	}
	
	@Override
    public boolean listenOn(String eventType) {
        return eventType.startsWith(FullTextSearchOperateEvent.class.getName());
    }
}

全文检索实体

java 复制代码 
@Entity
@Table(
    name="TV_FULLTEXT_SEARCH",
    indexes={
        @Index(name="idx_TV_FULLTEXT_SEARCH1",columnList="projKey"),
        @Index(name="idx_TV_FULLTEXT_SEARCH2",columnList="contentType")
    }
)
public class FullTextSearch extends IEntity {
	
    @Id
    @Column(length=200)
    private String id;
    private RelateProjValObj mainProj;//来源相关件
    @Lob
    @Type(type="org.hibernate.type.TextType")
    private String content;//检索内容
    @Column(length=100)
    private String contentType;//检索类型
    @Column(length=100)
    private Date lastUpdateDate;//最后更新时间
    
    
    public String getId() {
        return id;
    }
    public String getContent() {
        return content;
    }
    public String getContentType() {
        return contentType;
    }
    public RelateProjValObj getMainProj() {
        return mainProj;
    }
    public Date getLastUpdateDate() {
		return lastUpdateDate;
	}
	
	
	public FullTextSearch() {
    }
    public FullTextSearch(RelateProjValObj mainProj, String contentType,
    		String content, Date lastUpdateDate) {
        this.id = mainProj.getProjKey()+"_"+contentType;
        this.mainProj = mainProj;
        this.content = content;
        this.contentType = contentType;
        this.lastUpdateDate = lastUpdateDate;
        if(this.lastUpdateDate==null){
            this.lastUpdateDate = new Date();
        }
    }

}

存储数据格式


查询

sql大致就是这样的逻辑

sql 复制代码 
select tv.id from tv_cmp_dw_query tv join tv_fulltext_search tvs on tv.id = tvs.proj_id where tvs.contet_type in () and conent like '%测试%'

事件处理机制请看另一篇文章
自定义事件处理机制

相关推荐
问道飞鱼40 分钟前
【知识科普】认识正则表达式
数据库·mysql·正则表达式
HaiFan.1 小时前
SpringBoot 事务
java·数据库·spring boot·sql·mysql
水根LP491 小时前
linux系统上SQLPLUS的重“大”发现
数据库·oracle
途途途途1 小时前
精选9个自动化任务的Python脚本精选
数据库·python·自动化
04Koi.2 小时前
Redis--常用数据结构和编码方式
数据库·redis·缓存
silver98862 小时前
mongodb和Cassandra
数据库
PersistJiao2 小时前
3.基于 Temporal 的 Couchbase 动态 SQL 执行场景
数据库·sql
上山的月2 小时前
MySQL -函数和约束
数据库·mysql
zhcf3 小时前
【MySQL】十三，关于MySQL的全文索引
数据库·mysql
极限实验室3 小时前
Easysearch Chart Admin 密码自定义
数据库
热门推荐
01大模型国产化适配7-华为昇腾LLM落地可选解决方案（MindFormers、ModelLink、MindIE）02ARM学习（31）编译器对overlay方式的支持03玄机平台应急响应—webshell查杀04电赛2024年H题智能小车基于MSPM0G3507主控MCU（利用8路灰度加上MPU6050的解决方式）具体项目报告05【一文读懂】NTN（非地面网络）技术介绍06(欧拉)openEuler系统添加网卡文件配置流程、(欧拉)openEuler系统手动配置ipv6地址流程、(欧拉)openEuler系统网络管理说明07Ubuntu 20.04使用Livox mid 360 测试 FAST_LIO08组基轨迹建模 GBTM的介绍与实现（Stata 或 R）09Docker 夺命连环 15 问10STM32 无法烧录