开发背景
为什么要自研 ES-ORM？
在字节实习的时候，负责的一个项目叫做数据中心。 数据中心是呼叫中心部门呼叫服务的数据中枢，其核心分为两大模块，一为向业务方推送各种类型的消息体, 一为对外提供 统一的话单信息RPC查询服务。
这里 对外提供统一的话单信息RPC查询服务就需要去es里面根据各种条件做查询（话单类里面可能上百个字段，如果使用es原生api，那么将十分痛苦。。。）
所以实习期间搞了一套 ES-ORM 来简化代码开发。
同时 ES-ORM 可以屏蔽 ES-JAVA复杂的SDK。帮助大家更好的使用 ES 进行查询。只要你会使用 Mybatis-Plus 就能轻松使用 ES-ORM 对 ES进行查询
快速入门
实体类
实体类就是你要从es里面查询出来的类，在字节实习的时候 一条话单记录就是实体类。
这里使用 Student类 举例子。
java
@Data
@AllArgsConstructor
@NoArgsConstructor
@Index(indexName = "student")
public class Student {
@DocumentId(documentId = "studentId")
private Long studentId;
private String name;
private String school;
private Integer age;
private Integer classNo;
private Integer sex;
// 逻辑删除字段 isDelete 0 代表逻辑删除
@LogicDelete(filedName = "isDelete", deleteValue = 0)
private Integer isDelete;
}ps：
- @Index(indexName = "student") 注解指明了该实体类所在索引名字。如果不标注 @Index 注解，那么索引默认取类名小写，即 student
- @DocumentId 指定的是索引主键，如果不指定，默认取 "id"
- @LogicDelete(filedName = "isDelete", deleteValue = 0) 指定 isDelete 是逻辑删除字段，并且 delelteValue = 0 代表逻辑删除的值
StudentService
java
public class StudentService extends BaseService<Student> {
}你需要继承 BaseService，并且指定你要crud的类 Student。其中BaseService提供一些基础的 查询。 例如根据id查询，根据ids查询，根据条件查询一条数据or多条数据
使用示例
准备一个类 StudentQueryDTO ，用来接收查询Student的查询条件
java
@Data
@AllArgsConstructor
@Builder
public class StudentQueryDTO {
private Long id;
private String name;
private List<Object> names;
private Integer age;
private Integer fromAge;
private Integer toAge;
private List<Object> ages;
private String school;
private List<Integer> schools;
private Integer sex;
private Integer classNo;
private Integer fromClassNo;
private Integer toClassNo;
private List<Integer> classNos;
}根据ID查询单个Student
java
public static void testQueryById() {
/**
* equal sql :
* select * from student where studentId = 111
*/
Student student = studentService.getById(111);
}根据id查询列表 student
java
public static void testListByIds() {
/**
* equal sql :
* select * from student where studentId in (1, 2, 3)
*/
List<Student> students = studentService.listByIds(asList(1, 2, 3));
}普通查询
java
public static void testGetOneNormal() {
StudentQueryDTO queryDTO = StudentQueryDTO
.builder()
.age(15)
.build();
/**
* equal sql :
* select * from student where age = 15
*/
EsWrapper<Student> ageWrapper = of(Student.class).eq("age", 15);
Student student = studentService.getOne(ageWrapper);
}单个Wrapper之间的链式调用是 and 逻辑
java
public static void testGetOneNormal2() {
StudentQueryDTO queryDTO = StudentQueryDTO
.builder()
.age(15)
.name("qyh")
.build();
/**
* equal sql :
* select * from student where age = 15 and name = 'qyh'
*/
EsWrapper<Student> ageNameAndWrapper = of(Student.class)
.eq("age", queryDTO.getAge())
.eq("name", queryDTO.getName());
Student student = studentService.getOne(ageNameAndWrapper);
}指定需要查询的字段
java
public static void testGetOneNormal3() {
StudentQueryDTO queryDTO = StudentQueryDTO
.builder()
.age(15)
.name("qyh")
.build();
/**
* equal sql :
* select age, name from student where age = 15 and name = 'qyh'
*/
EsWrapper<Student> ageNameAndWrapper = of(Student.class)
.eq("age", queryDTO.getAge())
.eq("name", queryDTO.getName())
.includeFields(Student::getName, Student::getAge);
Student student = studentService.getOne(ageNameAndWrapper);
}conditionApi简化开发
java
public static void testGetOneConditionApi() {
StudentQueryDTO queryDTO = StudentQueryDTO
.builder()
.age(15)
.name("qyh")
.build();
/**
* equal sql :
* 1. queryDTO.age == null && queryDTO.name == null
* select * from student;
* 2. queryDTO.age == null && queryDTO.name != null
* select * from student where name = 'qyh'
* 3. queryDTO.age != null && queryDTO.name == null
* select * from student where age = 15
* 4. queryDTO.age != null && queryDTO.name != null
* select * from student where age = 15 and name = 'qyh'
*/
EsWrapper<Student> ageNameAndWrapper = of(Student.class)
.eq(nonNull(queryDTO.getAge()), "age", queryDTO.getAge())
.eq(nonNull(queryDTO.getName()), "name", queryDTO.getName());
Student student = studentService.getOne(ageNameAndWrapper);
}使用Lambda表达式屏蔽列名概念。
java
public static void testGetOneByLambdaApi() {
StudentQueryDTO queryDTO = StudentQueryDTO
.builder()
.age(15)
.name("qyh")
.build();
/**
* equal sql :
* 1. queryDTO.age == null && queryDTO.name == null
* select * from student;
* 2. queryDTO.age == null && queryDTO.name != null
* select * from student where name = 'qyh'
* 3. queryDTO.age != null && queryDTO.name == null
* select * from student where age = 15
* 4. queryDTO.age != null && queryDTO.name != null
* select * from student where age = 15 and name = 'qyh'
*/
EsWrapper<Student> ageNameAndWrapper = of(Student.class)
.eq(nonNull(queryDTO.getAge()), Student::getAge, queryDTO.getAge())
.eq(nonNull(queryDTO.getName()), Student::getName, queryDTO.getName());
Student student = studentService.getOne(ageNameAndWrapper);
}根据条件查询列表
java
public static void testListByCondition() {
/**
* equal sql :
* select * from student where studentId in (1, 2, 3)
*/
EsWrapper<Student> esWrapper = of(Student.class)
.in(Student::getStudentId, asList(1, 2, 3));
List<Student> res = studentService.list(esWrapper);
}指定 from 和 limit
java
public static void testPageListByCondition() {
/**
* equal sql :
* select * from student where studentId in (1, 2, 3) from 0 limit 3
*/
EsWrapper<Student> esWrapper = of(Student.class)
.in(Student::getStudentId, asList(1, 2, 3))
.from(0)
.size(3);
List<Student> res = studentService.list(esWrapper);
}自动过滤掉逻辑删除的数据
java
public static void testFilterLogicDelete() {
/**
* equal sql :
* select * from student where age in (1, 2, 3) and is_delete != 0 from 10 limit 20
*/
EsWrapper<Student> esWrapper = of(Student.class, FILTER_LOGIC_DELETE)
.in(Student::getAge, Lists.newArrayList(1, 2, 3))
.from(10)
.size(20);
List<Student> list = studentService.list(esWrapper);
}ps：这种方式需要在 Student 类的字段上添加注解 @LogicDelete 标明哪个是逻辑删除字段，并且指定逻辑删除的值
指定结果排序方式
java
public static void testOrderBy() {
/**
* equal sql :
* select * from student
* where age in (1, 2, 3) and is_delete != 0
* from 10 limit 20
* order by studentId desc
*/
EsWrapper<Student> esWrapper = of(Student.class, FILTER_LOGIC_DELETE)
.in(Student::getAge, Lists.newArrayList(1, 2, 3))
.from(10)
.size(20)
.orderBy(Student::getStudentId, DESC);
List<Student> list = studentService.list(esWrapper);
}复杂逻辑
这里通过 WrapperLogicOpHelper 可以构建 wrapper之间的任意亦或逻辑
java
public static void testLogicApi() {
StudentQueryDTO queryDTO = StudentQueryDTO
.builder()
.age(15)
.name("qyh")
.sex(1)
.build();
/**
* equal sql :
* select * from student where (age = 15 and name = 'qyh') or sex = 1
*/
EsWrapper<Student> ageNameAndWrapper = of(Student.class)
.eq(nonNull(queryDTO.getAge()), Student::getAge, queryDTO.getAge())
.eq(nonNull(queryDTO.getName()),Student::getName,queryDTO.getName());
EsWrapper<Student> sexWrapper = of(Student.class)
.eq(nonNull(queryDTO.getSex()), Student::getSex, queryDTO.getSex());
EsWrapper<Student> finalWrapper = EsWrapperLogicHelper.or(ageNameAndWrapper, sexWrapper);
Student student = studentService.getOne(finalWrapper);
}
public static void testLogicApi2() {
StudentQueryDTO queryDTO = StudentQueryDTO
.builder()
.age(15)
.name("qyh")
.sex(1)
.build();
/**
* equal sql :
* select * from student where (age = 15 or name = 'qyh') and sex = 1
*/
EsWrapper<Student> ageWrapper = of(Student.class)
.eq(nonNull(queryDTO.getAge()), Student::getAge, queryDTO.getAge());
EsWrapper<Student> nameWrapper = of(Student.class)
.eq(nonNull(queryDTO.getName()), Student::getName, queryDTO.getName());
EsWrapper<Student> ageNameOrWrapper = EsWrapperLogicHelper.or(ageWrapper, nameWrapper);
EsWrapper<Student> sexWrapper = of(Student.class)
.eq(nonNull(queryDTO.getSex()), Student::getSex, queryDTO.getSex());
EsWrapper<Student> finalWrapper = EsWrapperLogicHelper.and(ageNameOrWrapper, sexWrapper);
Student student = studentService.getOne(finalWrapper);
}
public static void testLogicApi3() {
StudentQueryDTO queryDTO = StudentQueryDTO
.builder()
.age(15)
.name("qyh")
.sex(1)
.classNo(14)
.build();
/**
* equal sql :
* select * from student where (age = 15 or name = 'qyh' or classNo = 14) and sex = 1
*/
EsWrapper<Student> ageWrapper = of(Student.class)
.eq(nonNull(queryDTO.getAge()), Student::getAge, queryDTO.getAge());
EsWrapper<Student> nameWrapper = of(Student.class)
.eq(nonNull(queryDTO.getName()), Student::getName, queryDTO.getName());
EsWrapper<Student> classNoWrapper = of(Student.class)
.eq(nonNull(queryDTO.getClassNo()), Student::getClassNo, queryDTO.getClassNo());
EsWrapper<Student> ageNameClassNoOrWrapper =
EsWrapperLogicHelper.or(ageWrapper, nameWrapper, classNoWrapper);
EsWrapper<Student> sexWrapper = of(Student.class)
.eq(nonNull(queryDTO.getSex()), Student::getSex, queryDTO.getSex());
EsWrapper<Student> finalWrapper =
EsWrapperLogicHelper.and(ageNameClassNoOrWrapper, sexWrapper);
Student student = studentService.getOne(finalWrapper);
}设计原理
泛型实例化
简而言之就是继承泛型类，然后指定泛型类型。然后就能拿到实例化的泛型，通过反射分析上面的注解，获得索引名称，主键字段。
下面是一个使用的例子。
java
public static void main(String[] args) {
StudentService studentService = new StudentService();
Student student = studentService.findById(1L);
TeacherService teacherService = new TeacherService();
Teacher teacher = teacherService.findById(2L);
//最后 生成的sql是 : select * from hdu_student where id = 1
//最后 生成的sql是 : select * from teacher where id = 2
}
static class IService<T> {
//获得实体类型
Class<T> getEntityClass() {
//获得子类 并获得子类所继承父类的泛型信息
Type genericSuperclass = this.getClass().getGenericSuperclass();
if (genericSuperclass instanceof ParameterizedType) {
ParameterizedType pt = (ParameterizedType) genericSuperclass;
Type actualTypeArgument = pt.getActualTypeArguments()[0];
return (Class<T>) actualTypeArgument;
} else {
throw new RuntimeException("没有获取到合适的泛型信息");
}
}
//根据id查询一条数据
T findById(Long id) {
String tableName = getTableName();
//最后查询的sql是
System.out.println("最后 生成的sql是 : select * from " + tableName + " where id = " + id);
//该方法还可以扩展 因为我们已经有 class信息了
//那么我们可以去获得各种各样的字段 ， 去解析字段上的注解......
return null;
}
//获得该实体类的表名
String getTableName() {
String tableName = null;
Class<?> entityClass = getEntityClass();
TableName tableNameAnnotation = entityClass.getAnnotation(TableName.class);
if (tableNameAnnotation != null) {
tableName = tableNameAnnotation.value();
} else {
//如果没有添加注解 那么就获得它的简单类名
//然后把首字母转化为小写 比如 Teacher -> teacher
tableName = entityClass.getSimpleName();
tableName = Character.toLowerCase(tableName.charAt(0)) + tableName.substring(1);
}
return tableName;
}
void save(T t) {
System.out.println("调用 ？ 类型的 save");
}
}
//已经确定的泛型
static class StudentService extends IService<Student> {
}
//已经确定的泛型
static class TeacherService extends IService<Teacher> {
}
}
//指定entity对应的表名是 hdu_student
@TableName("hdu_student")
class Student {
}
//@TableName("hdu_teacher")
class Teacher {
}
//可以放在实体类上面 指定他对应数据库的表名是什么
@Documented
@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.TYPE})
@interface TableName {
String value() default "";
}通过方法引用来屏蔽列名概念
java
public static void testGetOneByLambdaApi() {
StudentQueryDTO queryDTO = StudentQueryDTO
.builder()
.age(15)
.name("qyh")
.build();
/**
* equal sql :
* 1. queryDTO.age == null && queryDTO.name == null
* select * from student;
* 2. queryDTO.age == null && queryDTO.name != null
* select * from student where name = 'qyh'
* 3. queryDTO.age != null && queryDTO.name == null
* select * from student where age = 15
* 4. queryDTO.age != null && queryDTO.name != null
* select * from student where age = 15 and name = 'qyh'
*/
Wrapper ageNameAndWrapper = new LambdaEsWrapper<Student>()
.eq(nonNull(queryDTO.getAge()), Student::getAge, 15)
.eq(nonNull(queryDTO.getName()), Student::getName, "qyh");
Student student = studentService.getOne(ageNameAndWrapper);
}上面传参 Student::getAge, Student::getName 。会被底层接收为 SFunction(一个可以序列化的Lambda表达式）
然后其中的 implMethodName 属性 就是调用的方法名，再取 getXxx 的 Xxx部分就是属性名，这样就完成了对列名概念的屏蔽。
底层核心模型 QueryFilter
在字节实习的时候，搞了一套 JavaBean -> QueryDSL 的转化模型。下面举个例子来说明一下该模型。
java
public class QueryFilter {
String field;
String exp;
FilterType type;
List<QueryFilter> must;
List<QueryFilter> mustNot;
List<QueryFilter> should;
List<Object> in;
List<Object> notIn;
Object eq;
Object notEq;
Object gt;
Object gte;
Object lt;
Object lte;
}
public enum FilterType {
// 嵌套
NESTED,
// 等于
EQ,
// 不等于
NOT_EQ,
// 属于
IN,
// 不属于
NOT_IN,
// 范围
RANGE,
// 存在
EXISTS,
// 不存在
NOT_EXISTS,
// 脚本表达式
EXP
}例子
QueryFilter -> BoolQueryBuilder 的转换
java
public static QueryBuilder buildQueryBuilder(QueryFilter queryFilter) {
if (queryFilter == null) {
return null;
}
FilterType type = queryFilter.getType();
String field = queryFilter.getField();
final BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
switch (type) {
case NESTED:
ofNullable(queryFilter.getMust()).ifPresent(
mustList -> {
for (QueryFilter must : mustList) {
ofNullable(buildQueryBuilder(must)).ifPresent(boolQuery::must);
}
}
);
ofNullable(queryFilter.getMustNot()).ifPresent(
mustNotList -> {
for (QueryFilter mustNot : mustNotList) {
ofNullable(buildQueryBuilder(mustNot)).ifPresent(boolQuery::mustNot);
}
}
);
ofNullable(queryFilter.getShould()).ifPresent(
shouldList -> {
for (QueryFilter should : shouldList) {
ofNullable(buildQueryBuilder(should)).ifPresent(boolQuery::should);
}
/**
使用should查询的时候，且除了should查询之外的还有must，filter中的任意一个的时候，should条件默认会被自动忽略
*/
boolQuery.minimumShouldMatch(1);
}
);
return boolQuery;
case EQ:
return QueryBuilders.termQuery(field, queryFilter.getEq());
case NOT_EQ:
boolQuery.mustNot(termQuery(field, queryFilter.getNotEq()));
return boolQuery;
case RANGE:
RangeQueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery(field);
ofNullable(queryFilter.getGt()).ifPresent(rangeQueryBuilder::gt);
ofNullable(queryFilter.getGte()).ifPresent(rangeQueryBuilder::gte);
ofNullable(queryFilter.getLt()).ifPresent(rangeQueryBuilder::lt);
ofNullable(queryFilter.getLte()).ifPresent(rangeQueryBuilder::lte);
return rangeQueryBuilder;
case IN:
return termsQuery(field, queryFilter.getIn());
case NOT_IN:
boolQuery.mustNot(termsQuery(field, queryFilter.getNotIn()));
return boolQuery;
case EXISTS:
return existsQuery(field);
case NOT_EXISTS:
boolQuery.mustNot(existsQuery(field));
return boolQuery;
case EXP:
return QueryBuilders.scriptQuery(new Script(queryFilter.getExp()));
default:
throw new UnsupportedOperationException();
}
}链式调用修改QueryFilter
举个例子
每个 Wrapper 底层都有一个基础 QueryFilter 。基础 QueryFilter类型是 NESTED 类型，它有一个must数组。每次链式调用其实都是往 基础 QueryFilter里面的 must数组塞各种类型的QueryFilter，那么我们也可以得知：单个 Wrapper之间各种链式调用逻辑之间是且的关系。下面拿eq（）方法举个例子。
java
// where age = 15
of(Student.class).eq(nonNull(queryDTO.getAge(), "age", 15);
public NormalEsWrapper<T> eq(boolean condition, String field, Object eq) {
if (condition) {
// 构建 age = 15 的 QueryFilter -> eqQueryFilter
QueryFilter eqQueryFilter = new QueryFilter();
eqQueryFilter.setType(FilterType.EQ);
eqQueryFilter.setField(field);
eqQueryFilter.setEq(eq);
// 获得最上层 QueryFilter
List<QueryFilter> must = this.getQueryFilter().getMust();
// 获得其must数组（对应的就是 && 逻辑）
must = ofNullable(must).orElseGet(ArrayList::new);
// 将 eqQueryFilter 添加到 最上层的 QueryFilter的 must数组
// 从这里看出来 单个Wrapper链式调用之间的亦或逻辑是 &&
must.add(eqQueryFilter);
this.getQueryFilter().setMust(must);
}
// 链式调用返回
return this;
}真实例子
java
public static void testLogicApi3() {
StudentQueryDTO queryDTO = StudentQueryDTO
.builder()
.age(15)
.name("qyh")
.sex(1)
.classNo(14)
.build();
/**
* equal sql :
* select * from student where (age = 15 or name = 'qyh' or classNo = 14) and sex = 1
*/
EsWrapper<Student> ageWrapper = of(Student.class)
.eq(nonNull(queryDTO.getAge()), Student::getAge, queryDTO.getAge());
EsWrapper<Student> nameWrapper = of(Student.class)
.eq(nonNull(queryDTO.getName()), Student::getName, queryDTO.getName());
EsWrapper<Student> classNoWrapper = of(Student.class)
.eq(nonNull(queryDTO.getClassNo()), Student::getClassNo, queryDTO.getClassNo());
EsWrapper<Student> ageNameClassNoOrWrapper =
EsWrapperLogicHelper.or(ageWrapper, nameWrapper, classNoWrapper);
EsWrapper<Student> sexWrapper = of(Student.class)
.eq(nonNull(queryDTO.getSex()), Student::getSex, queryDTO.getSex());
EsWrapper<Student> finalWrapper =
EsWrapperLogicHelper.and(ageNameClassNoOrWrapper, sexWrapper);
Student student = studentService.getOne(finalWrapper);
}最终 finalWrapper长这样
finalWrapper对应的QueryFilter
json
{
"type": "NESTED",
"must": [{
"type": "NESTED",
"should": [{
"type": "NESTED",
"must": [{
"eq": 15,
"type": "EQ",
"field": "age"
}]
}, {
"type": "NESTED",
"must": [{
"eq": "qyh",
"type": "EQ",
"field": "name"
}]
}, {
"type": "NESTED",
"must": [{
"eq": 14,
"type": "EQ",
"field": "classNo"
}]
}]
}, {
"type": "NESTED",
"must": [{
"eq": 1,
"type": "EQ",
"field": "sex"
}]
}]
}